nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "lexer_t.h"
   4 #include "token_t.h"
   5 #include "symbol_table_t.h"
   6 #include "adt/error.h"
   7
   8 #include <assert.h>
   9 #include <errno.h>
  10 #include <string.h>
  11 #include <ctype.h>
  12
  13 //#define DEBUG_CHARS
  14 #define MAX_PUTBACK 3
  15
  16 static
  17 void error_prefix_at(lexer_t *this, const char *input_name, unsigned linenr)
  18 {
  19         (void) this;
  20         fprintf(stderr, "%s:%d: Error: ", input_name, linenr);
  21 }
  22
  23 static
  24 void error_prefix(lexer_t *this)
  25 {
  26         error_prefix_at(this, this->source_position.input_name,
  27                         this->source_position.linenr);
  28 }
  29
  30 static
  31 void parse_error(lexer_t *this, const char *msg)
  32 {
  33         error_prefix(this);
  34         fprintf(stderr, "%s\n", msg);
  35 }
  36
  37 static inline
  38 void next_char(lexer_t *this)
  39 {
  40         this->bufpos++;
  41         if(this->bufpos >= this->bufend) {
  42                 size_t s = fread(this->buf + MAX_PUTBACK, 1,
  43                                  sizeof(this->buf) - MAX_PUTBACK, this->input);
  44                 if(s == 0) {
  45                         this->c = EOF;
  46                         return;
  47                 }
  48                 this->bufpos = this->buf + MAX_PUTBACK;
  49                 this->bufend = this->buf + MAX_PUTBACK + s;
  50         }
  51         this->c = *(this->bufpos);
  52 #ifdef DEBUG_CHARS
  53         printf("nchar '%c'\n", this->c);
  54 #endif
  55 }
  56
  57 static inline
  58 void put_back(lexer_t *this, int c)
  59 {
  60         char *p = (char*) this->bufpos - 1;
  61         this->bufpos--;
  62         assert(p >= this->buf);
  63         *p = c;
  64
  65 #ifdef DEBUG_CHARS
  66         printf("putback '%c'\n", c);
  67 #endif
  68 }
  69
  70
  71 static
  72 int replace_trigraph(lexer_t *this)
  73 {
  74 #define MATCH_TRIGRAPH(ch,replacement)           \
  75         case ch:                                     \
  76                 this->c = replacement;                   \
  77                 return 1;
  78
  79         switch(this->c) {
  80         MATCH_TRIGRAPH('=', '#')
  81         MATCH_TRIGRAPH('(', '[')
  82         MATCH_TRIGRAPH('/', '\\')
  83         MATCH_TRIGRAPH(')', ']')
  84         MATCH_TRIGRAPH('\'', '^')
  85         MATCH_TRIGRAPH('<', '{')
  86         MATCH_TRIGRAPH('!', '|')
  87         MATCH_TRIGRAPH('>', '}')
  88         MATCH_TRIGRAPH('-', '~')
  89         default:
  90                 break;
  91         }
  92
  93         return 0;
  94 }
  95
  96 #define SKIP_TRIGRAPHS(custom_putback, no_trigraph_code) \
  97         case '?':                                  \
  98                 next_char(this);                       \
  99                 if(this->c != '?') {                   \
 100                         custom_putback;                    \
 101                         put_back(this, this->c);           \
 102                         this->c = '?';                     \
 103                         no_trigraph_code;                  \
 104                 }                                      \
 105                 next_char(this);                       \
 106                 if(replace_trigraph(this)) {           \
 107                         break;                             \
 108                 }                                      \
 109                 custom_putback;                        \
 110                 put_back(this, '?');                   \
 111                 put_back(this, this->c);               \
 112                 this->c = '?';                         \
 113                 no_trigraph_code;
 114
 115 #define EAT_NEWLINE(newline_code)              \
 116         if(this->c == '\r') {                      \
 117                 next_char(this);                       \
 118                 if(this->c == '\n')                    \
 119                         next_char(this);                   \
 120                 this->source_position.linenr++;        \
 121                 newline_code;                          \
 122         } else if(this->c == '\n') {               \
 123                 next_char(this);                       \
 124                 this->source_position.linenr++;        \
 125                 newline_code;                          \
 126         }
 127
 128 static
 129 void parse_symbol(lexer_t *this, token_t *token)
 130 {
 131         symbol_t *symbol;
 132         char     *string;
 133
 134         obstack_1grow(&symbol_obstack, this->c);
 135         next_char(this);
 136
 137         while(1) {
 138                 switch(this->c) {
 139                 case '\\':
 140                         next_char(this);
 141                         EAT_NEWLINE(break;)
 142                         goto end_symbol;
 143
 144                 case 'A' ... 'Z':
 145                 case 'a' ... 'z':
 146                 case '_':
 147                         obstack_1grow(&symbol_obstack, this->c);
 148                         next_char(this);
 149                         break;
 150
 151                 case '?':
 152                         next_char(this);
 153                         if(this->c != '?') {
 154                                 put_back(this, this->c);
 155                                 this->c = '?';
 156                                 goto end_symbol;
 157                         }
 158                         next_char(this);
 159                         if(replace_trigraph(this))
 160                                 break;
 161                         put_back(this, '?');
 162                         put_back(this, this->c);
 163                         this->c = '?';
 164                         goto end_symbol;
 165
 166                 default:
 167                         goto end_symbol;
 168                 }
 169         }
 170 end_symbol:
 171         obstack_1grow(&symbol_obstack, '\0');
 172
 173         string = obstack_finish(&symbol_obstack);
 174         symbol = symbol_table_insert(string);
 175
 176         if(symbol->ID > 0) {
 177                 token->type = symbol->ID;
 178         } else {
 179                 token->type = T_IDENTIFIER;
 180         }
 181         token->v.symbol = symbol;
 182
 183         if(symbol->string != string) {
 184                 obstack_free(&symbol_obstack, string);
 185         }
 186 }
 187
 188 #if 0
 189 static
 190 preprocessor_token_type_t parse_pp_symbol(lexer_t *this)
 191 {
 192         do {
 193                 obstack_1grow(&symbol_obstack, this->c);
 194                 next_char(this);
 195         } while(is_ident_char(this->c));
 196         obstack_1grow(&symbol_obstack, '\0');
 197
 198         char     *string = obstack_finish(&symbol_obstack);
 199         symbol_t *symbol = preprocessor_symbol_table_find(string);
 200         obstack_free(&symbol_obstack, string);
 201
 202         if(symbol == 0)
 203                 return TP_ERROR;
 204
 205         return symbol->ID;
 206 }
 207 #endif
 208
 209 static
 210 void parse_number_hex(lexer_t *this, token_t *token)
 211 {
 212         assert(this->c == 'x' || this->c == 'X');
 213         next_char(this);
 214
 215         if (!isdigit(this->c) &&
 216                 !('A' <= this->c && this->c <= 'F') &&
 217                 !('a' <= this->c && this->c <= 'f')) {
 218                 parse_error(this, "premature end of hex number literal");
 219                 token->type = T_ERROR;
 220                 return;
 221         }
 222
 223         int value = 0;
 224         for(;;) {
 225                 if (isdigit(this->c)) {
 226                         value = 16 * value + this->c - '0';
 227                 } else if ('A' <= this->c && this->c <= 'F') {
 228                         value = 16 * value + this->c - 'A' + 10;
 229                 } else if ('a' <= this->c && this->c <= 'f') {
 230                         value = 16 * value + this->c - 'a' + 10;
 231                 } else {
 232                         token->type     = T_INTEGER;
 233                         token->v.intvalue = value;
 234                         return;
 235                 }
 236                 next_char(this);
 237         }
 238 }
 239
 240 static
 241 void parse_number_oct(lexer_t *this, token_t *token)
 242 {
 243         assert(this->c == 'o' || this->c == 'O');
 244         next_char(this);
 245
 246         int value = 0;
 247         for(;;) {
 248                 if ('0' <= this->c && this->c <= '7') {
 249                         value = 8 * value + this->c - '0';
 250                 } else {
 251                         token->type     = T_INTEGER;
 252                         token->v.intvalue = value;
 253                         return;
 254                 }
 255                 next_char(this);
 256         }
 257 }
 258
 259 static
 260 void parse_number_dec(lexer_t *this, token_t *token, int first_char)
 261 {
 262         int value = 0;
 263         if(first_char > 0) {
 264                 assert(first_char >= '0' && first_char <= '9');
 265                 value = first_char - '0';
 266         }
 267
 268         for(;;) {
 269                 if (isdigit(this->c)) {
 270                         value = 10 * value + this->c - '0';
 271                 } else {
 272                         token->type     = T_INTEGER;
 273                         token->v.intvalue = value;
 274                         return;
 275                 }
 276                 next_char(this);
 277         }
 278 }
 279
 280 static
 281 void parse_number(lexer_t *this, token_t *token)
 282 {
 283         // TODO check for overflow
 284         // TODO check for various invalid inputs sequences
 285
 286         if (this->c == '0') {
 287                 next_char(this);
 288                 switch (this->c) {
 289                         case 'X':
 290                         case 'x': parse_number_hex(this, token); break;
 291                         case 'o':
 292                         case 'O': parse_number_oct(this, token); break;
 293                         default:  parse_number_dec(this, token, '0');
 294                 }
 295         } else {
 296                 parse_number_dec(this, token, 0);
 297         }
 298 }
 299
 300 static
 301 int parse_escape_sequence(lexer_t *this)
 302 {
 303         while(1) {
 304                 int c = this->c;
 305                 next_char(this);
 306
 307                 switch(c) {
 308                 case '"': return '"';
 309                 case '\'': return'\'';
 310                 case '\\':
 311                         EAT_NEWLINE(break;)
 312                         return '\\';
 313                 case 'a': return '\a';
 314                 case 'b': return '\b';
 315                 case 'f': return '\f';
 316                 case 'n': return '\n';
 317                 case 'r': return '\r';
 318                 case 't': return '\t';
 319                 case 'v': return '\v';
 320                 case 'x': /* TODO parse hex number ... */
 321                         parse_error(this, "hex escape sequences not implemented yet");
 322                         return EOF;
 323                 case 0 ... 8: /* TODO parse octal number ... */
 324                         parse_error(this, "octal escape sequences not implemented yet");
 325                         return EOF;
 326                 case '?':
 327                         if(this->c != '?') {
 328                                 return '?';
 329                         }
 330                         /* might be a trigraph */
 331                         next_char(this);
 332                         if(replace_trigraph(this)) {
 333                                 break;
 334                         }
 335                         put_back(this, this->c);
 336                         this->c = '?';
 337                         return '?';
 338
 339                 case EOF:
 340                         parse_error(this, "reached end of file while parsing escape sequence");
 341                         return EOF;
 342                 default:
 343                         parse_error(this, "unknown escape sequence");
 344                         return EOF;
 345                 }
 346         }
 347 }
 348
 349 static
 350 void parse_string_literal(lexer_t *this, token_t *token)
 351 {
 352         unsigned    start_linenr = this->source_position.linenr;
 353         char       *string;
 354         const char *result;
 355
 356         assert(this->c == '"');
 357         next_char(this);
 358
 359         while(1) {
 360                 switch(this->c) {
 361                 SKIP_TRIGRAPHS(,
 362                         obstack_1grow(&symbol_obstack, '?');
 363                         next_char(this);
 364                         break;
 365                 )
 366
 367                 case '\\':
 368                         next_char(this);
 369                         EAT_NEWLINE(break;)
 370                         int c = parse_escape_sequence(this);
 371                         obstack_1grow(&symbol_obstack, c);
 372                         break;
 373
 374                 case EOF:
 375                         error_prefix_at(this, this->source_position.input_name,
 376                                         start_linenr);
 377                         fprintf(stderr, "string has no end\n");
 378                         token->type = T_ERROR;
 379                         return;
 380
 381                 case '"':
 382                         next_char(this);
 383                         goto end_of_string;
 384
 385                 default:
 386                         obstack_1grow(&symbol_obstack, this->c);
 387                         next_char(this);
 388                         break;
 389                 }
 390         }
 391
 392 end_of_string:
 393
 394         /* TODO: concatenate multiple strings separated by whitespace... */
 395
 396         /* add finishing 0 to the string */
 397         obstack_1grow(&symbol_obstack, '\0');
 398         string = obstack_finish(&symbol_obstack);
 399
 400         /* check if there is already a copy of the string */
 401         result = strset_insert(&this->stringset, string);
 402         if(result != string) {
 403                 obstack_free(&symbol_obstack, string);
 404         }
 405
 406         token->type     = T_STRING_LITERAL;
 407         token->v.string = result;
 408 }
 409
 410 #define MATCH_NEWLINE(code)                 \
 411         case '\r':                              \
 412                 next_char(this);                    \
 413                 if(this->c == '\n') {               \
 414                         next_char(this);                \
 415                 }                                   \
 416                 this->source_position.linenr++;     \
 417                 code;                               \
 418         case '\n':                              \
 419                 next_char(this);                    \
 420                 this->source_position.linenr++;     \
 421                 code;
 422
 423 static
 424 void parse_character_constant(lexer_t *this, token_t *token)
 425 {
 426         assert(this->c == '\'');
 427         next_char(this);
 428
 429         int found_char = 0;
 430         while(1) {
 431                 switch(this->c) {
 432                 SKIP_TRIGRAPHS(,
 433                         found_char = '?';
 434                         break;
 435                 )
 436
 437                 case '\\':
 438                         next_char(this);
 439                         EAT_NEWLINE(break;)
 440                         found_char = '\\';
 441                         break;
 442
 443                 MATCH_NEWLINE(
 444                         parse_error(this, "newline while parsing character constant");
 445                         break;
 446                 )
 447
 448                 case '\'':
 449                         next_char(this);
 450                         goto end_of_char_constant;
 451
 452                 case EOF:
 453                         parse_error(this, "EOF while parsing character constant");
 454                         token->type = T_ERROR;
 455                         return;
 456
 457                 default:
 458                         if(found_char != 0) {
 459                                 parse_error(this, "more than 1 characters in character "
 460                                             "constant");
 461                                 goto end_of_char_constant;
 462                         } else {
 463                                 found_char = this->c;
 464                                 next_char(this);
 465                         }
 466                         break;
 467                 }
 468         }
 469
 470 end_of_char_constant:
 471         token->type       = T_INTEGER;
 472         token->v.intvalue = found_char;
 473 }
 474
 475 static
 476 void skip_multiline_comment(lexer_t *this)
 477 {
 478         unsigned start_linenr = this->source_position.linenr;
 479         int had_star = 0;
 480
 481         while(1) {
 482                 switch(this->c) {
 483                 case '*':
 484                         next_char(this);
 485                         had_star = 1;
 486                         break;
 487
 488                 case '/':
 489                         next_char(this);
 490                         if(had_star) {
 491                                 return;
 492                         }
 493                         had_star = 0;
 494                         break;
 495
 496                 case '\\':
 497                         next_char(this);
 498                         EAT_NEWLINE(break;)
 499                         had_star = 0;
 500                         break;
 501
 502                 case '?':
 503                         next_char(this);
 504                         if(this->c != '?') {
 505                                 had_star = 0;
 506                                 break;
 507                         }
 508                         next_char(this);
 509                         if(replace_trigraph(this))
 510                                 break;
 511                         put_back(this, this->c);
 512                         this->c = '?';
 513                         had_star = 0;
 514                         /* we don't put back the 2nd ? as the comment text is discarded
 515                          * anyway */
 516                         break;
 517
 518                 MATCH_NEWLINE(had_star = 0; break;)
 519
 520                 case EOF:
 521                         error_prefix_at(this, this->source_position.input_name,
 522                                         start_linenr);
 523                         fprintf(stderr, "at end of file while looking for comment end\n");
 524                         return;
 525                 default:
 526                         had_star = 0;
 527                         next_char(this);
 528                         break;
 529                 }
 530         }
 531 }
 532
 533 static
 534 void skip_line_comment(lexer_t *this)
 535 {
 536         while(1) {
 537                 switch(this->c) {
 538                 case '?':
 539                         next_char(this);
 540                         if(this->c != '?')
 541                                 break;
 542                         next_char(this);
 543                         if(replace_trigraph(this))
 544                                 break;
 545                         put_back(this, '?');
 546                         /* we don't put back the 2nd ? as the comment text is discarded
 547                          * anyway */
 548                         break;
 549
 550                 case '\\':
 551                         next_char(this);
 552                         if(this->c == '\n') {
 553                                 next_char(this);
 554                                 this->source_position.linenr++;
 555                         }
 556                         break;
 557
 558                 case EOF:
 559                 case '\r':
 560                 case '\n':
 561                         return;
 562
 563                 default:
 564                         next_char(this);
 565                         break;
 566                 }
 567         }
 568 }
 569
 570 static
 571 void parse_preprocessor_directive(lexer_t *this, token_t *result_token)
 572 {
 573         printf("PP: ");
 574         while(this->c != '\n') {
 575                 printf("%c", this->c);
 576                 next_char(this);
 577         }
 578         printf("\n");
 579
 580         lexer_next_token(this, result_token);
 581 }
 582
 583 void preprocessor_next_token(lexer_t *this, token_t *token)
 584 {
 585         /* skip whitespaces */
 586         while(this->c == ' ' || this->c == '\t' || this->c == '\r') {
 587                 next_char(this);
 588         }
 589
 590         switch(this->c) {
 591         case 'A' ... 'Z':
 592         case 'a' ... 'z':
 593         case '_':
 594                 parse_symbol(this, token);
 595         }
 596 }
 597
 598 #define MAYBE_PROLOG                                       \
 599                         next_char(this);                               \
 600                         while(1) {                                     \
 601                                 switch(this->c) {
 602
 603 #define MAYBE(ch, set_type)                                \
 604                                 case ch:                                   \
 605                                         next_char(this);                       \
 606                                         token->type = set_type;                \
 607                                         return;
 608
 609 #define ELSE_CODE(code)                                    \
 610                                 SKIP_TRIGRAPHS(,                           \
 611                                         code;                                  \
 612                                 )                                          \
 613                                                                                                                    \
 614                                 case '\\':                                 \
 615                                         next_char(this);                       \
 616                                         EAT_NEWLINE(break;)                    \
 617                                         /* fallthrough */                      \
 618                                 default:                                   \
 619                                         code;                                  \
 620                                 }                                          \
 621                         } /* end of while(1) */                        \
 622                         break;
 623
 624 #define ELSE(set_type)                                     \
 625                 ELSE_CODE(                                         \
 626                         token->type = set_type;                        \
 627                         return;                                        \
 628                 )
 629
 630 static
 631 void eat_whitespace(lexer_t *this)
 632 {
 633         while(1) {
 634                 switch(this->c) {
 635                 case ' ':
 636                 case '\t':
 637                         next_char(this);
 638                         break;
 639
 640                 MATCH_NEWLINE(
 641                         break;
 642                 )
 643
 644                 case '\\':
 645                         next_char(this);
 646                         if(this->c == '\n') {
 647                                 next_char(this);
 648                                 this->source_position.linenr++;
 649                                 break;
 650                         }
 651
 652                         put_back(this, this->c);
 653                         this->c = '\\';
 654                         return;
 655
 656                 SKIP_TRIGRAPHS(,
 657                         return;
 658                 )
 659
 660                 case '/':
 661                         next_char(this);
 662                         while(1) {
 663                                 switch(this->c) {
 664                                 case '*':
 665                                         next_char(this);
 666                                         skip_multiline_comment(this);
 667                                         eat_whitespace(this);
 668                                         return;
 669                                 case '/':
 670                                         next_char(this);
 671                                         skip_line_comment(this);
 672                                         eat_whitespace(this);
 673                                         return;
 674
 675                                 SKIP_TRIGRAPHS(
 676                                                 put_back(this, '?');
 677                                         ,
 678                                                 this->c = '/';
 679                                                 return;
 680                                 )
 681
 682                                 case '\\':
 683                                         next_char(this);
 684                                         EAT_NEWLINE(break;)
 685                                         /* fallthrough */
 686                                 default:
 687                                         return;
 688                                 }
 689                         }
 690                         break;
 691
 692                 default:
 693                         return;
 694                 }
 695         }
 696 }
 697
 698 void lexer_next_token(lexer_t *this, token_t *token)
 699 {
 700         while(1) {
 701                 switch(this->c) {
 702                 case ' ':
 703                 case '\t':
 704                         next_char(this);
 705                         break;
 706
 707                 MATCH_NEWLINE(
 708                         eat_whitespace(this);
 709                         if(this->c == '#') {
 710                                 next_char(this);
 711                                 parse_preprocessor_directive(this, token);
 712                                 return;
 713                         }
 714                         break;
 715                 )
 716
 717                 case 'A' ... 'Z':
 718                 case 'a' ... 'z':
 719                 case '_':
 720                         parse_symbol(this, token);
 721                         return;
 722
 723                 case '0' ... '9':
 724                         parse_number(this, token);
 725                         return;
 726
 727                 case '"':
 728                         parse_string_literal(this, token);
 729                         return;
 730
 731                 case '\'':
 732                         parse_character_constant(this, token);
 733                         return;
 734
 735                 case '\\':
 736                         next_char(this);
 737                         if(this->c == '\n') {
 738                                 next_char(this);
 739                                 this->source_position.linenr++;
 740                                 break;
 741                         } else {
 742                                 parse_error(this, "unexpected '\\' found");
 743                                 token->type = T_ERROR;
 744                         }
 745                         return;
 746
 747                 case '.':
 748                         MAYBE_PROLOG
 749                                 case '.':
 750                                         MAYBE_PROLOG
 751                                         MAYBE('.', T_DOTDOTDOT)
 752                                         ELSE_CODE(
 753                                                 put_back(this, this->c);
 754                                                 this->c = '.';
 755                                                 token->type = '.';
 756                                                 return;
 757                                         )
 758                         ELSE('.')
 759                 case '&':
 760                         MAYBE_PROLOG
 761                         MAYBE('&', T_ANDAND)
 762                         MAYBE('=', T_ANDEQUAL)
 763                         ELSE('&')
 764                 case '*':
 765                         MAYBE_PROLOG
 766                         MAYBE('=', T_ASTERISKEQUAL)
 767                         ELSE('*')
 768                 case '+':
 769                         MAYBE_PROLOG
 770                         MAYBE('+', T_PLUSPLUS)
 771                         MAYBE('=', T_PLUSEQUAL)
 772                         ELSE('+')
 773                 case '-':
 774                         MAYBE_PROLOG
 775                         MAYBE('-', T_MINUSMINUS)
 776                         MAYBE('=', T_MINUSEQUAL)
 777                         ELSE('-')
 778                 case '!':
 779                         MAYBE_PROLOG
 780                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
 781                         ELSE('!')
 782                 case '/':
 783                         MAYBE_PROLOG
 784                         MAYBE('=', T_SLASHEQUAL)
 785                                 case '*':
 786                                         next_char(this);
 787                                         skip_multiline_comment(this);
 788                                         lexer_next_token(this, token);
 789                                         return;
 790                                 case '/':
 791                                         next_char(this);
 792                                         skip_line_comment(this);
 793                                         lexer_next_token(this, token);
 794                                         return;
 795                         ELSE('/')
 796                 case '%':
 797                         MAYBE_PROLOG
 798                         MAYBE('>', T_PERCENTGREATER)
 799                         MAYBE('=', T_PERCENTEQUAL)
 800                                 case ':':
 801                                         MAYBE_PROLOG
 802                                                 case '%':
 803                                                         MAYBE_PROLOG
 804                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
 805                                                         ELSE_CODE(
 806                                                                 put_back(this, this->c);
 807                                                                 this->c = '%';
 808                                                                 token->type = T_PERCENTCOLON;
 809                                                                 return;
 810                                                         )
 811                                         ELSE(T_PERCENTCOLON)
 812                         ELSE('%')
 813                 case '<':
 814                         MAYBE_PROLOG
 815                         MAYBE(':', T_LESSCOLON)
 816                         MAYBE('%', T_LESSPERCENT)
 817                                 case '<':
 818                                         MAYBE_PROLOG
 819                                         MAYBE('=', T_LESSLESSEQUAL)
 820                                         ELSE(T_LESSLESS)
 821                         ELSE('<')
 822                 case '>':
 823                         MAYBE_PROLOG
 824                                 case '>':
 825                                         MAYBE_PROLOG
 826                                         MAYBE('=', T_GREATERGREATEREQUAL)
 827                                         ELSE(T_GREATERGREATER)
 828                         ELSE('>')
 829                 case '^':
 830                         MAYBE_PROLOG
 831                         MAYBE('=', T_CARETEQUAL)
 832                         ELSE('^')
 833                 case '|':
 834                         MAYBE_PROLOG
 835                         MAYBE('=', T_PIPEEQUAL)
 836                         MAYBE('|', T_PIPEPIPE)
 837                         ELSE('|')
 838                 case ':':
 839                         MAYBE_PROLOG
 840                         MAYBE('>', T_COLONGREATER)
 841                         ELSE(':')
 842                 case '=':
 843                         MAYBE_PROLOG
 844                         MAYBE('=', T_EQUALEQUAL)
 845                         ELSE('=')
 846                 case '#':
 847                         MAYBE_PROLOG
 848                         MAYBE('#', T_HASHHASH)
 849                         ELSE('#')
 850
 851                 case '?':
 852                         next_char(this);
 853                         /* just a simple ? */
 854                         if(this->c != '?') {
 855                                 token->type = '?';
 856                                 return;
 857                         }
 858                         /* might be a trigraph */
 859                         next_char(this);
 860                         if(replace_trigraph(this)) {
 861                                 break;
 862                         }
 863                         put_back(this, this->c);
 864                         this->c = '?';
 865                         token->type = '?';
 866                         return;
 867
 868                 case '[':
 869                 case ']':
 870                 case '(':
 871                 case ')':
 872                 case '{':
 873                 case '}':
 874                 case '~':
 875                 case ';':
 876                 case ',':
 877                         token->type = this->c;
 878                         next_char(this);
 879                         return;
 880
 881                 case EOF:
 882                         token->type = T_EOF;
 883                         return;
 884
 885                 default:
 886                         next_char(this);
 887                         error_prefix(this);
 888                         fprintf(stderr, "unknown character '%c' found\n", this->c);
 889                         token->type = T_ERROR;
 890                         return;
 891                 }
 892         }
 893 }
 894
 895 void lexer_init(lexer_t *this, FILE *stream, const char *input_name)
 896 {
 897         memset(this, 0, sizeof(this[0]));
 898
 899         this->input = stream;
 900
 901         this->source_position.linenr     = 0;
 902         this->source_position.input_name = input_name;
 903         strset_init(&this->stringset);
 904
 905         /* we place a virtual '\n' at the beginning so the lexer knows we're at the
 906          * beginning of a line */
 907         this->c = '\n';
 908 }
 909
 910 void lexer_destroy(lexer_t *this)
 911 {
 912         (void) this;
 913 }
 914
 915 static __attribute__((unused))
 916 void dbg_pos(const source_position_t source_position)
 917 {
 918         fprintf(stdout, "%s:%d\n", source_position.input_name, source_position.linenr);
 919         fflush(stdout);
 920 }