nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 //#define DEBUG_CHARS
  44 #define MAX_PUTBACK 3
  45
  46 #ifdef _WIN32
  47 /* No strtold on windows and no replacement yet */
  48 #define strtold(s, e) strtod(s, e)
  49 #endif
  50
  51 static int         c;
  52 token_t            lexer_token;
  53 symbol_t          *symbol_L;
  54 static FILE       *input;
  55 static char        buf[1024 + MAX_PUTBACK];
  56 static const char *bufend;
  57 static const char *bufpos;
  58 static strset_t    stringset;
  59
  60 /**
  61  * Prints a parse error message at the current token.
  62  *
  63  * @param msg   the error message
  64  */
  65 static void parse_error(const char *msg)
  66 {
  67         errorf(&lexer_token.source_position,  "%s", msg);
  68 }
  69
  70 /**
  71  * Prints an internal error message at the current token.
  72  *
  73  * @param msg   the error message
  74  */
  75 static NORETURN internal_error(const char *msg)
  76 {
  77         internal_errorf(&lexer_token.source_position,  "%s", msg);
  78 }
  79
  80 static inline void next_real_char(void)
  81 {
  82         assert(bufpos <= bufend);
  83         if (bufpos >= bufend) {
  84                 if (input == NULL) {
  85                         c = EOF;
  86                         return;
  87                 }
  88
  89                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  90                                  input);
  91                 if(s == 0) {
  92                         c = EOF;
  93                         return;
  94                 }
  95                 bufpos = buf + MAX_PUTBACK;
  96                 bufend = buf + MAX_PUTBACK + s;
  97         }
  98         c = *bufpos++;
  99 }
 100
 101 /**
 102  * Put a character back into the buffer.
 103  *
 104  * @param pc  the character to put back
 105  */
 106 static inline void put_back(int pc)
 107 {
 108         assert(bufpos > buf);
 109         *(--bufpos - buf + buf) = (char) pc;
 110
 111 #ifdef DEBUG_CHARS
 112         printf("putback '%c'\n", pc);
 113 #endif
 114 }
 115
 116 static inline void next_char(void);
 117
 118 #define MATCH_NEWLINE(code)                   \
 119         case '\r':                                \
 120                 next_char();                          \
 121                 if(c == '\n') {                       \
 122                         next_char();                      \
 123                 }                                     \
 124                 lexer_token.source_position.linenr++; \
 125                 code                                  \
 126         case '\n':                                \
 127                 next_char();                          \
 128                 lexer_token.source_position.linenr++; \
 129                 code
 130
 131 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 132
 133 static void maybe_concat_lines(void)
 134 {
 135         eat('\\');
 136
 137         switch(c) {
 138         MATCH_NEWLINE(return;)
 139
 140         default:
 141                 break;
 142         }
 143
 144         put_back(c);
 145         c = '\\';
 146 }
 147
 148 /**
 149  * Set c to the next input character, ie.
 150  * after expanding trigraphs.
 151  */
 152 static inline void next_char(void)
 153 {
 154         next_real_char();
 155
 156         /* filter trigraphs */
 157         if(UNLIKELY(c == '\\')) {
 158                 maybe_concat_lines();
 159                 goto end_of_next_char;
 160         }
 161
 162         if(LIKELY(c != '?'))
 163                 goto end_of_next_char;
 164
 165         next_real_char();
 166         if(LIKELY(c != '?')) {
 167                 put_back(c);
 168                 c = '?';
 169                 goto end_of_next_char;
 170         }
 171
 172         next_real_char();
 173         switch(c) {
 174         case '=': c = '#'; break;
 175         case '(': c = '['; break;
 176         case '/': c = '\\'; maybe_concat_lines(); break;
 177         case ')': c = ']'; break;
 178         case '\'': c = '^'; break;
 179         case '<': c = '{'; break;
 180         case '!': c = '|'; break;
 181         case '>': c = '}'; break;
 182         case '-': c = '~'; break;
 183         default:
 184                 put_back(c);
 185                 put_back('?');
 186                 c = '?';
 187                 break;
 188         }
 189
 190 end_of_next_char:;
 191 #ifdef DEBUG_CHARS
 192         printf("nchar '%c'\n", c);
 193 #endif
 194 }
 195
 196 #define SYMBOL_CHARS  \
 197         case 'a':         \
 198         case 'b':         \
 199         case 'c':         \
 200         case 'd':         \
 201         case 'e':         \
 202         case 'f':         \
 203         case 'g':         \
 204         case 'h':         \
 205         case 'i':         \
 206         case 'j':         \
 207         case 'k':         \
 208         case 'l':         \
 209         case 'm':         \
 210         case 'n':         \
 211         case 'o':         \
 212         case 'p':         \
 213         case 'q':         \
 214         case 'r':         \
 215         case 's':         \
 216         case 't':         \
 217         case 'u':         \
 218         case 'v':         \
 219         case 'w':         \
 220         case 'x':         \
 221         case 'y':         \
 222         case 'z':         \
 223         case 'A':         \
 224         case 'B':         \
 225         case 'C':         \
 226         case 'D':         \
 227         case 'E':         \
 228         case 'F':         \
 229         case 'G':         \
 230         case 'H':         \
 231         case 'I':         \
 232         case 'J':         \
 233         case 'K':         \
 234         case 'L':         \
 235         case 'M':         \
 236         case 'N':         \
 237         case 'O':         \
 238         case 'P':         \
 239         case 'Q':         \
 240         case 'R':         \
 241         case 'S':         \
 242         case 'T':         \
 243         case 'U':         \
 244         case 'V':         \
 245         case 'W':         \
 246         case 'X':         \
 247         case 'Y':         \
 248         case 'Z':         \
 249         case '_':
 250
 251 #define DIGITS        \
 252         case '0':         \
 253         case '1':         \
 254         case '2':         \
 255         case '3':         \
 256         case '4':         \
 257         case '5':         \
 258         case '6':         \
 259         case '7':         \
 260         case '8':         \
 261         case '9':
 262
 263 /**
 264  * Read a symbol from the input and build
 265  * the lexer_token.
 266  */
 267 static void parse_symbol(void)
 268 {
 269         symbol_t *symbol;
 270         char     *string;
 271
 272         obstack_1grow(&symbol_obstack, (char) c);
 273         next_char();
 274
 275         while(1) {
 276                 switch(c) {
 277                 DIGITS
 278                 SYMBOL_CHARS
 279                         obstack_1grow(&symbol_obstack, (char) c);
 280                         next_char();
 281                         break;
 282
 283                 default:
 284                         goto end_symbol;
 285                 }
 286         }
 287
 288 end_symbol:
 289         obstack_1grow(&symbol_obstack, '\0');
 290
 291         string = obstack_finish(&symbol_obstack);
 292         symbol = symbol_table_insert(string);
 293
 294         lexer_token.type     = symbol->ID;
 295         lexer_token.v.symbol = symbol;
 296
 297         if(symbol->string != string) {
 298                 obstack_free(&symbol_obstack, string);
 299         }
 300 }
 301
 302 static void parse_integer_suffix(bool is_oct_hex)
 303 {
 304         bool is_unsigned  = false;
 305         bool min_long     = false;
 306         bool min_longlong = false;
 307
 308         if(c == 'U' || c == 'u') {
 309                 is_unsigned = true;
 310                 next_char();
 311                 if(c == 'L' || c == 'l') {
 312                         min_long = true;
 313                         next_char();
 314                         if(c == 'L' || c == 'l') {
 315                                 min_longlong = true;
 316                                 next_char();
 317                         }
 318                 }
 319         } else if(c == 'l' || c == 'L') {
 320                 min_long = true;
 321                 next_char();
 322                 if(c == 'l' || c == 'L') {
 323                         min_longlong = true;
 324                         next_char();
 325                         if(c == 'u' || c == 'U') {
 326                                 is_unsigned = true;
 327                                 next_char();
 328                         }
 329                 } else if(c == 'u' || c == 'U') {
 330                         is_unsigned = true;
 331                         next_char();
 332                         lexer_token.datatype = type_unsigned_long;
 333                 }
 334         }
 335
 336         if(!is_unsigned) {
 337                 long long v = lexer_token.v.intvalue;
 338                 if(!min_long) {
 339                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 340                                 lexer_token.datatype = type_int;
 341                                 return;
 342                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 343                                 lexer_token.datatype = type_unsigned_int;
 344                                 return;
 345                         }
 346                 }
 347                 if(!min_longlong) {
 348                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 349                                 lexer_token.datatype = type_long;
 350                                 return;
 351                         } else if(is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) {
 352                                 lexer_token.datatype = type_unsigned_long;
 353                                 return;
 354                         }
 355                 }
 356                 unsigned long long uv = (unsigned long long) v;
 357                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 358                         lexer_token.datatype = type_unsigned_long_long;
 359                         return;
 360                 }
 361
 362                 lexer_token.datatype = type_long_long;
 363         } else {
 364                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 365                 if(!min_long && v <= TARGET_UINT_MAX) {
 366                         lexer_token.datatype = type_unsigned_int;
 367                         return;
 368                 }
 369                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 370                         lexer_token.datatype = type_unsigned_long;
 371                         return;
 372                 }
 373                 lexer_token.datatype = type_unsigned_long_long;
 374         }
 375 }
 376
 377 static void parse_floating_suffix(void)
 378 {
 379         switch(c) {
 380         /* TODO: do something useful with the suffixes... */
 381         case 'f':
 382         case 'F':
 383                 next_char();
 384                 lexer_token.datatype = type_float;
 385                 break;
 386         case 'l':
 387         case 'L':
 388                 next_char();
 389                 lexer_token.datatype = type_long_double;
 390                 break;
 391         default:
 392                 lexer_token.datatype = type_double;
 393                 break;
 394         }
 395 }
 396
 397 /**
 398  * A replacement for strtoull. Only those parts needed for
 399  * our parser are implemented.
 400  */
 401 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 402         unsigned long long v = 0;
 403
 404         switch (base) {
 405         case 16:
 406                 for (;; ++s) {
 407                         /* check for overrun */
 408                         if (v >= 0x1000000000000000ULL)
 409                                 break;
 410                         switch (tolower(*s)) {
 411                         case '0': v <<= 4; break;
 412                         case '1': v <<= 4; v |= 0x1; break;
 413                         case '2': v <<= 4; v |= 0x2; break;
 414                         case '3': v <<= 4; v |= 0x3; break;
 415                         case '4': v <<= 4; v |= 0x4; break;
 416                         case '5': v <<= 4; v |= 0x5; break;
 417                         case '6': v <<= 4; v |= 0x6; break;
 418                         case '7': v <<= 4; v |= 0x7; break;
 419                         case '8': v <<= 4; v |= 0x8; break;
 420                         case '9': v <<= 4; v |= 0x9; break;
 421                         case 'a': v <<= 4; v |= 0xa; break;
 422                         case 'b': v <<= 4; v |= 0xb; break;
 423                         case 'c': v <<= 4; v |= 0xc; break;
 424                         case 'd': v <<= 4; v |= 0xd; break;
 425                         case 'e': v <<= 4; v |= 0xe; break;
 426                         case 'f': v <<= 4; v |= 0xf; break;
 427                         default:
 428                                 goto end;
 429                         }
 430                 }
 431                 break;
 432         case 8:
 433                 for (;; ++s) {
 434                         /* check for overrun */
 435                         if (v >= 0x2000000000000000ULL)
 436                                 break;
 437                         switch (tolower(*s)) {
 438                         case '0': v <<= 3; break;
 439                         case '1': v <<= 3; v |= 1; break;
 440                         case '2': v <<= 3; v |= 2; break;
 441                         case '3': v <<= 3; v |= 3; break;
 442                         case '4': v <<= 3; v |= 4; break;
 443                         case '5': v <<= 3; v |= 5; break;
 444                         case '6': v <<= 3; v |= 6; break;
 445                         case '7': v <<= 3; v |= 7; break;
 446                         default:
 447                                 goto end;
 448                         }
 449                 }
 450                 break;
 451         case 10:
 452                 for (;; ++s) {
 453                         /* check for overrun */
 454                         if (v > 0x1999999999999999ULL)
 455                                 break;
 456                         switch (tolower(*s)) {
 457                         case '0': v *= 10; break;
 458                         case '1': v *= 10; v += 1; break;
 459                         case '2': v *= 10; v += 2; break;
 460                         case '3': v *= 10; v += 3; break;
 461                         case '4': v *= 10; v += 4; break;
 462                         case '5': v *= 10; v += 5; break;
 463                         case '6': v *= 10; v += 6; break;
 464                         case '7': v *= 10; v += 7; break;
 465                         case '8': v *= 10; v += 8; break;
 466                         case '9': v *= 10; v += 9; break;
 467                         default:
 468                                 goto end;
 469                         }
 470                 }
 471                 break;
 472         default:
 473                 assert(0);
 474                 break;
 475         }
 476 end:
 477         *endptr = s;
 478         return v;
 479 }
 480
 481 /**
 482  * Parses a hex number including hex floats and set the
 483  * lexer_token.
 484  */
 485 static void parse_number_hex(void)
 486 {
 487         assert(c == 'x' || c == 'X');
 488         next_char();
 489
 490         while(isxdigit(c)) {
 491                 obstack_1grow(&symbol_obstack, (char) c);
 492                 next_char();
 493         }
 494         obstack_1grow(&symbol_obstack, '\0');
 495         char *string = obstack_finish(&symbol_obstack);
 496
 497         if(c == '.' || c == 'p' || c == 'P') {
 498                 next_char();
 499                 internal_error("Hex floating point numbers not implemented yet");
 500         }
 501         if(*string == '\0') {
 502                 parse_error("invalid hex number");
 503                 lexer_token.type = T_ERROR;
 504         }
 505
 506         const char *endptr;
 507         lexer_token.type       = T_INTEGER;
 508         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 509         if(*endptr != '\0') {
 510                 parse_error("hex number literal too long");
 511         }
 512
 513         obstack_free(&symbol_obstack, string);
 514         parse_integer_suffix(true);
 515 }
 516
 517 /**
 518  * Returns true if the given char is a octal digit.
 519  *
 520  * @param char  the character to check
 521  */
 522 static inline bool is_octal_digit(int chr)
 523 {
 524         switch(chr) {
 525         case '0':
 526         case '1':
 527         case '2':
 528         case '3':
 529         case '4':
 530         case '5':
 531         case '6':
 532         case '7':
 533                 return true;
 534         default:
 535                 return false;
 536         }
 537 }
 538
 539 /**
 540  * Parses a octal number and set the lexer_token.
 541  */
 542 static void parse_number_oct(void)
 543 {
 544         while(is_octal_digit(c)) {
 545                 obstack_1grow(&symbol_obstack, (char) c);
 546                 next_char();
 547         }
 548         obstack_1grow(&symbol_obstack, '\0');
 549         char *string = obstack_finish(&symbol_obstack);
 550
 551         const char *endptr;
 552         lexer_token.type       = T_INTEGER;
 553         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 554         if(*endptr != '\0') {
 555                 parse_error("octal number literal too long");
 556         }
 557
 558         obstack_free(&symbol_obstack, string);
 559         parse_integer_suffix(true);
 560 }
 561
 562 /**
 563  * Parses a decimal including float number and set the
 564  * lexer_token.
 565  */
 566 static void parse_number_dec(void)
 567 {
 568         bool is_float = false;
 569         while(isdigit(c)) {
 570                 obstack_1grow(&symbol_obstack, (char) c);
 571                 next_char();
 572         }
 573
 574         if(c == '.') {
 575                 obstack_1grow(&symbol_obstack, '.');
 576                 next_char();
 577
 578                 while(isdigit(c)) {
 579                         obstack_1grow(&symbol_obstack, (char) c);
 580                         next_char();
 581                 }
 582                 is_float = true;
 583         }
 584         if(c == 'e' || c == 'E') {
 585                 obstack_1grow(&symbol_obstack, 'e');
 586                 next_char();
 587
 588                 if(c == '-' || c == '+') {
 589                         obstack_1grow(&symbol_obstack, (char) c);
 590                         next_char();
 591                 }
 592
 593                 while(isdigit(c)) {
 594                         obstack_1grow(&symbol_obstack, (char) c);
 595                         next_char();
 596                 }
 597                 is_float = true;
 598         }
 599
 600         obstack_1grow(&symbol_obstack, '\0');
 601         char *string = obstack_finish(&symbol_obstack);
 602
 603         if(is_float) {
 604                 char *endptr;
 605                 lexer_token.type         = T_FLOATINGPOINT;
 606                 lexer_token.v.floatvalue = strtold(string, &endptr);
 607
 608                 if(*endptr != '\0') {
 609                         parse_error("invalid number literal");
 610                 }
 611
 612                 parse_floating_suffix();
 613         } else {
 614                 const char *endptr;
 615                 lexer_token.type       = T_INTEGER;
 616                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 617
 618                 if(*endptr != '\0') {
 619                         parse_error("invalid number literal");
 620                 }
 621
 622                 parse_integer_suffix(false);
 623         }
 624         obstack_free(&symbol_obstack, string);
 625 }
 626
 627 /**
 628  * Parses a number and sets the lexer_token.
 629  */
 630 static void parse_number(void)
 631 {
 632         if (c == '0') {
 633                 next_char();
 634                 switch (c) {
 635                         case 'X':
 636                         case 'x':
 637                                 parse_number_hex();
 638                                 break;
 639                         case '0':
 640                         case '1':
 641                         case '2':
 642                         case '3':
 643                         case '4':
 644                         case '5':
 645                         case '6':
 646                         case '7':
 647                                 parse_number_oct();
 648                                 break;
 649                         case '8':
 650                         case '9':
 651                                 next_char();
 652                                 parse_error("invalid octal number");
 653                                 lexer_token.type = T_ERROR;
 654                                 return;
 655                         case '.':
 656                         case 'e':
 657                         case 'E':
 658                         default:
 659                                 obstack_1grow(&symbol_obstack, '0');
 660                                 parse_number_dec();
 661                                 return;
 662                 }
 663         } else {
 664                 parse_number_dec();
 665         }
 666 }
 667
 668 /**
 669  * Returns the value of a digit.
 670  * The only portable way to do it ...
 671  */
 672 static int digit_value(int digit) {
 673         switch (digit) {
 674         case '0': return 0;
 675         case '1': return 1;
 676         case '2': return 2;
 677         case '3': return 3;
 678         case '4': return 4;
 679         case '5': return 5;
 680         case '6': return 6;
 681         case '7': return 7;
 682         case '8': return 8;
 683         case '9': return 9;
 684         case 'a':
 685         case 'A': return 10;
 686         case 'b':
 687         case 'B': return 11;
 688         case 'c':
 689         case 'C': return 12;
 690         case 'd':
 691         case 'D': return 13;
 692         case 'e':
 693         case 'E': return 14;
 694         case 'f':
 695         case 'F': return 15;
 696         default:
 697                 internal_error("wrong character given");
 698         }
 699 }
 700
 701 /**
 702  * Parses an octal character sequence.
 703  *
 704  * @param first_digit  the already read first digit
 705  */
 706 static int parse_octal_sequence(const int first_digit)
 707 {
 708         assert(is_octal_digit(first_digit));
 709         int value = digit_value(first_digit);
 710         if (!is_octal_digit(c)) return value;
 711         value = 8 * value + digit_value(c);
 712         next_char();
 713         if (!is_octal_digit(c)) return value;
 714         value = 8 * value + digit_value(c);
 715         next_char();
 716
 717         if(char_is_signed) {
 718                 return (signed char) value;
 719         } else {
 720                 return (unsigned char) value;
 721         }
 722 }
 723
 724 /**
 725  * Parses a hex character sequence.
 726  */
 727 static int parse_hex_sequence(void)
 728 {
 729         int value = 0;
 730         while(isxdigit(c)) {
 731                 value = 16 * value + digit_value(c);
 732                 next_char();
 733         }
 734
 735         if(char_is_signed) {
 736                 return (signed char) value;
 737         } else {
 738                 return (unsigned char) value;
 739         }
 740 }
 741
 742 /**
 743  * Parse an escape sequence.
 744  */
 745 static int parse_escape_sequence(void)
 746 {
 747         eat('\\');
 748
 749         int ec = c;
 750         next_char();
 751
 752         switch(ec) {
 753         case '"':  return '"';
 754         case '\'': return '\'';
 755         case '\\': return '\\';
 756         case '?': return '\?';
 757         case 'a': return '\a';
 758         case 'b': return '\b';
 759         case 'f': return '\f';
 760         case 'n': return '\n';
 761         case 'r': return '\r';
 762         case 't': return '\t';
 763         case 'v': return '\v';
 764         case 'x':
 765                 return parse_hex_sequence();
 766         case '0':
 767         case '1':
 768         case '2':
 769         case '3':
 770         case '4':
 771         case '5':
 772         case '6':
 773         case '7':
 774                 return parse_octal_sequence(ec);
 775         case EOF:
 776                 parse_error("reached end of file while parsing escape sequence");
 777                 return EOF;
 778         default:
 779                 parse_error("unknown escape sequence");
 780                 return EOF;
 781         }
 782 }
 783
 784 /**
 785  * Concatenate two strings.
 786  */
 787 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 788 {
 789         const size_t len1 = s1->size - 1;
 790         const size_t len2 = s2->size - 1;
 791
 792         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 793         memcpy(concat, s1->begin, len1);
 794         memcpy(concat + len1, s2->begin, len2 + 1);
 795
 796 #if 0 /* TODO hash */
 797         const char *result = strset_insert(&stringset, concat);
 798         if(result != concat) {
 799                 obstack_free(&symbol_obstack, concat);
 800         }
 801
 802         return result;
 803 #else
 804         return (string_t){ concat, len1 + len2 + 1 };
 805 #endif
 806 }
 807
 808 /**
 809  * Concatenate a string and a wide string.
 810  */
 811 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 812 {
 813         const size_t len1 = s1->size - 1;
 814         const size_t len2 = s2->size - 1;
 815
 816         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 817         const char *const src = s1->begin;
 818         for (size_t i = 0; i != len1; ++i) {
 819                 concat[i] = src[i];
 820         }
 821         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 822
 823         return (wide_string_t){ concat, len1 + len2 + 1 };
 824 }
 825
 826 /**
 827  * Concatenate two wide strings.
 828  */
 829 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 830 {
 831         const size_t len1 = s1->size - 1;
 832         const size_t len2 = s2->size - 1;
 833
 834         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 835         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 836         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 837
 838         return (wide_string_t){ concat, len1 + len2 + 1 };
 839 }
 840
 841 /**
 842  * Concatenate a wide string and a string.
 843  */
 844 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 845 {
 846         const size_t len1 = s1->size - 1;
 847         const size_t len2 = s2->size - 1;
 848
 849         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 850         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 851         const char *const src = s2->begin;
 852         for (size_t i = 0; i != len2 + 1; ++i) {
 853                 concat[i] = src[i];
 854         }
 855
 856         return (wide_string_t){ concat, len1 + len2 + 1 };
 857 }
 858
 859 /**
 860  * Parse a string literal and set lexer_token.
 861  */
 862 static void parse_string_literal(void)
 863 {
 864         const unsigned start_linenr = lexer_token.source_position.linenr;
 865
 866         eat('"');
 867
 868         int tc;
 869         while(1) {
 870                 switch(c) {
 871                 case '\\':
 872                         tc = parse_escape_sequence();
 873                         obstack_1grow(&symbol_obstack, (char) tc);
 874                         break;
 875
 876                 case EOF: {
 877                         source_position_t source_position;
 878                         source_position.input_name = lexer_token.source_position.input_name;
 879                         source_position.linenr     = start_linenr;
 880                         errorf(&source_position, "string has no end");
 881                         lexer_token.type = T_ERROR;
 882                         return;
 883                 }
 884
 885                 case '"':
 886                         next_char();
 887                         goto end_of_string;
 888
 889                 default:
 890                         obstack_1grow(&symbol_obstack, (char) c);
 891                         next_char();
 892                         break;
 893                 }
 894         }
 895
 896 end_of_string:
 897
 898         /* TODO: concatenate multiple strings separated by whitespace... */
 899
 900         /* add finishing 0 to the string */
 901         obstack_1grow(&symbol_obstack, '\0');
 902         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 903         const char *const string = obstack_finish(&symbol_obstack);
 904
 905 #if 0 /* TODO hash */
 906         /* check if there is already a copy of the string */
 907         result = strset_insert(&stringset, string);
 908         if(result != string) {
 909                 obstack_free(&symbol_obstack, string);
 910         }
 911 #else
 912         const char *const result = string;
 913 #endif
 914
 915         lexer_token.type           = T_STRING_LITERAL;
 916         lexer_token.v.string.begin = result;
 917         lexer_token.v.string.size  = size;
 918 }
 919
 920 /**
 921  * Parse a wide character constant and set lexer_token.
 922  */
 923 static void parse_wide_character_constant(void)
 924 {
 925         const unsigned start_linenr = lexer_token.source_position.linenr;
 926
 927         eat('\'');
 928
 929         while(1) {
 930                 switch(c) {
 931                 case '\\': {
 932                         wchar_rep_t tc = parse_escape_sequence();
 933                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 934                         break;
 935                 }
 936
 937                 MATCH_NEWLINE(
 938                         parse_error("newline while parsing character constant");
 939                         break;
 940                 )
 941
 942                 case '\'':
 943                         next_char();
 944                         goto end_of_wide_char_constant;
 945
 946                 case EOF: {
 947                         source_position_t source_position = lexer_token.source_position;
 948                         source_position.linenr = start_linenr;
 949                         errorf(&source_position, "EOF while parsing character constant");
 950                         lexer_token.type = T_ERROR;
 951                         return;
 952                 }
 953
 954                 default: {
 955                         wchar_rep_t tc = (wchar_rep_t) c;
 956                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 957                         next_char();
 958                         break;
 959                 }
 960                 }
 961         }
 962
 963 end_of_wide_char_constant:;
 964         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
 965         assert(size % sizeof(wchar_rep_t) == 0);
 966         size /= sizeof(wchar_rep_t);
 967
 968         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
 969
 970         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
 971         lexer_token.v.wide_string.begin = string;
 972         lexer_token.v.wide_string.size  = size;
 973         lexer_token.datatype            = type_wchar_t;
 974 }
 975
 976 /**
 977  * Parse a wide string literal and set lexer_token.
 978  */
 979 static void parse_wide_string_literal(void)
 980 {
 981         const unsigned start_linenr = lexer_token.source_position.linenr;
 982
 983         assert(c == '"');
 984         next_char();
 985
 986         while(1) {
 987                 switch(c) {
 988                 case '\\': {
 989                         wchar_rep_t tc = parse_escape_sequence();
 990                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 991                         break;
 992                 }
 993
 994                 case EOF: {
 995                         source_position_t source_position;
 996                         source_position.input_name = lexer_token.source_position.input_name;
 997                         source_position.linenr     = start_linenr;
 998                         errorf(&source_position, "string has no end");
 999                         lexer_token.type = T_ERROR;
1000                         return;
1001                 }
1002
1003                 case '"':
1004                         next_char();
1005                         goto end_of_string;
1006
1007                 default: {
1008                         wchar_rep_t tc = c;
1009                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1010                         next_char();
1011                         break;
1012                 }
1013                 }
1014         }
1015
1016 end_of_string:;
1017
1018         /* TODO: concatenate multiple strings separated by whitespace... */
1019
1020         /* add finishing 0 to the string */
1021         wchar_rep_t nul = L'\0';
1022         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1023         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1024         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1025
1026 #if 0 /* TODO hash */
1027         /* check if there is already a copy of the string */
1028         const wchar_rep_t *const result = strset_insert(&stringset, string);
1029         if(result != string) {
1030                 obstack_free(&symbol_obstack, string);
1031         }
1032 #else
1033         const wchar_rep_t *const result = string;
1034 #endif
1035
1036         lexer_token.type                = T_WIDE_STRING_LITERAL;
1037         lexer_token.v.wide_string.begin = result;
1038         lexer_token.v.wide_string.size  = size;
1039 }
1040
1041 /**
1042  * Parse a character constant and set lexer_token.
1043  */
1044 static void parse_character_constant(void)
1045 {
1046         const unsigned start_linenr = lexer_token.source_position.linenr;
1047
1048         eat('\'');
1049
1050         while(1) {
1051                 switch(c) {
1052                 case '\\': {
1053                         int tc = parse_escape_sequence();
1054                         obstack_1grow(&symbol_obstack, (char) tc);
1055                         break;
1056                 }
1057
1058                 MATCH_NEWLINE(
1059                         parse_error("newline while parsing character constant");
1060                         break;
1061                 )
1062
1063                 case '\'':
1064                         next_char();
1065                         goto end_of_char_constant;
1066
1067                 case EOF: {
1068                         source_position_t source_position;
1069                         source_position.input_name = lexer_token.source_position.input_name;
1070                         source_position.linenr     = start_linenr;
1071                         errorf(&source_position, "EOF while parsing character constant");
1072                         lexer_token.type = T_ERROR;
1073                         return;
1074                 }
1075
1076                 default:
1077                         obstack_1grow(&symbol_obstack, (char) c);
1078                         next_char();
1079                         break;
1080
1081                 }
1082         }
1083
1084 end_of_char_constant:;
1085         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1086         const char *const string = obstack_finish(&symbol_obstack);
1087
1088         lexer_token.type           = T_CHARACTER_CONSTANT;
1089         lexer_token.v.string.begin = string;
1090         lexer_token.v.string.size  = size;
1091         lexer_token.datatype       = type_int;
1092 }
1093
1094 /**
1095  * Skip a multiline comment.
1096  */
1097 static void skip_multiline_comment(void)
1098 {
1099         unsigned start_linenr = lexer_token.source_position.linenr;
1100
1101         while(1) {
1102                 switch(c) {
1103                 case '/':
1104                         next_char();
1105                         if (c == '*') {
1106                                 /* TODO: nested comment, warn here */
1107                         }
1108                         break;
1109                 case '*':
1110                         next_char();
1111                         if(c == '/') {
1112                                 next_char();
1113                                 return;
1114                         }
1115                         break;
1116
1117                 MATCH_NEWLINE(break;)
1118
1119                 case EOF: {
1120                         source_position_t source_position;
1121                         source_position.input_name = lexer_token.source_position.input_name;
1122                         source_position.linenr     = start_linenr;
1123                         errorf(&source_position, "at end of file while looking for comment end");
1124                         return;
1125                 }
1126
1127                 default:
1128                         next_char();
1129                         break;
1130                 }
1131         }
1132 }
1133
1134 /**
1135  * Skip a single line comment.
1136  */
1137 static void skip_line_comment(void)
1138 {
1139         while(1) {
1140                 switch(c) {
1141                 case EOF:
1142                         return;
1143
1144                 case '\n':
1145                 case '\r':
1146                         return;
1147
1148                 default:
1149                         next_char();
1150                         break;
1151                 }
1152         }
1153 }
1154
1155 /** The current preprocessor token. */
1156 static token_t pp_token;
1157
1158 /**
1159  * Read the next preprocessor token.
1160  */
1161 static inline void next_pp_token(void)
1162 {
1163         lexer_next_preprocessing_token();
1164         pp_token = lexer_token;
1165 }
1166
1167 /**
1168  * Eat all preprocessor tokens until newline.
1169  */
1170 static void eat_until_newline(void)
1171 {
1172         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1173                 next_pp_token();
1174         }
1175 }
1176
1177 /**
1178  * Handle the define directive.
1179  */
1180 static void define_directive(void)
1181 {
1182         lexer_next_preprocessing_token();
1183         if(lexer_token.type != T_IDENTIFIER) {
1184                 parse_error("expected identifier after #define\n");
1185                 eat_until_newline();
1186         }
1187 }
1188
1189 /**
1190  * Handle the ifdef directive.
1191  */
1192 static void ifdef_directive(int is_ifndef)
1193 {
1194         (void) is_ifndef;
1195         lexer_next_preprocessing_token();
1196         //expect_identifier();
1197         //extect_newline();
1198 }
1199
1200 /**
1201  * Handle the endif directive.
1202  */
1203 static void endif_directive(void)
1204 {
1205         //expect_newline();
1206 }
1207
1208 /**
1209  * Parse the line directive.
1210  */
1211 static void parse_line_directive(void)
1212 {
1213         if(pp_token.type != T_INTEGER) {
1214                 parse_error("expected integer");
1215         } else {
1216                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1217                 next_pp_token();
1218         }
1219         if(pp_token.type == T_STRING_LITERAL) {
1220                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1221                 next_pp_token();
1222         }
1223
1224         eat_until_newline();
1225 }
1226
1227 /**
1228  * STDC pragmas.
1229  */
1230 typedef enum stdc_pragma_kind_t {
1231         STDC_UNKNOWN,
1232         STDC_FP_CONTRACT,
1233         STDC_FENV_ACCESS,
1234         STDC_CX_LIMITED_RANGE
1235 } stdc_pragma_kind_t;
1236
1237 /**
1238  * STDC pragma values.
1239  */
1240 typedef enum stdc_pragma_value_kind_t {
1241         STDC_VALUE_UNKNOWN,
1242         STDC_VALUE_ON,
1243         STDC_VALUE_OFF,
1244         STDC_VALUE_DEFAULT
1245 } stdc_pragma_value_kind_t;
1246
1247 /**
1248  * Parse a pragma directive.
1249  */
1250 static void parse_pragma(void) {
1251         bool unknown_pragma = true;
1252
1253         next_pp_token();
1254         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1255                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1256                 /* a STDC pragma */
1257                 if (c_mode & _C99) {
1258                         next_pp_token();
1259
1260                         switch (pp_token.v.symbol->pp_ID) {
1261                         case TP_FP_CONTRACT:
1262                                 kind = STDC_FP_CONTRACT;
1263                                 break;
1264                         case TP_FENV_ACCESS:
1265                                 kind = STDC_FENV_ACCESS;
1266                                 break;
1267                         case TP_CX_LIMITED_RANGE:
1268                                 kind = STDC_CX_LIMITED_RANGE;
1269                                 break;
1270                         default:
1271                                 break;
1272                         }
1273                         if (kind != STDC_UNKNOWN) {
1274                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1275                                 next_pp_token();
1276                                 switch (pp_token.v.symbol->pp_ID) {
1277                                 case TP_ON:
1278                                         value = STDC_VALUE_ON;
1279                                         break;
1280                                 case TP_OFF:
1281                                         value = STDC_VALUE_OFF;
1282                                         break;
1283                                 case TP_DEFAULT:
1284                                         value = STDC_VALUE_DEFAULT;
1285                                         break;
1286                                 default:
1287                                         break;
1288                                 }
1289                                 if (value != STDC_VALUE_UNKNOWN) {
1290                                         unknown_pragma = false;
1291                                 } else {
1292                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1293                                 }
1294                         }
1295                 }
1296         } else {
1297                 unknown_pragma = true;
1298         }
1299         eat_until_newline();
1300         if (unknown_pragma && warning.unknown_pragmas) {
1301                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1302         }
1303 }
1304
1305 /**
1306  * Parse a preprocessor non-null directive.
1307  */
1308 static void parse_preprocessor_identifier(void)
1309 {
1310         assert(pp_token.type == T_IDENTIFIER);
1311         symbol_t *symbol = pp_token.v.symbol;
1312
1313         switch(symbol->pp_ID) {
1314         case TP_include:
1315                 printf("include - enable header name parsing!\n");
1316                 break;
1317         case TP_define:
1318                 define_directive();
1319                 break;
1320         case TP_ifdef:
1321                 ifdef_directive(0);
1322                 break;
1323         case TP_ifndef:
1324                 ifdef_directive(1);
1325                 break;
1326         case TP_endif:
1327                 endif_directive();
1328                 break;
1329         case TP_line:
1330                 next_pp_token();
1331                 parse_line_directive();
1332                 break;
1333         case TP_if:
1334         case TP_else:
1335         case TP_elif:
1336         case TP_undef:
1337         case TP_error:
1338                 /* TODO; output the rest of the line */
1339                 parse_error("#error directive: ");
1340                 break;
1341         case TP_pragma:
1342                 parse_pragma();
1343                 break;
1344         }
1345 }
1346
1347 /**
1348  * Parse a preprocessor directive.
1349  */
1350 static void parse_preprocessor_directive(void)
1351 {
1352         next_pp_token();
1353
1354         switch(pp_token.type) {
1355         case T_IDENTIFIER:
1356                 parse_preprocessor_identifier();
1357                 break;
1358         case T_INTEGER:
1359                 parse_line_directive();
1360                 break;
1361         case '\n':
1362                 /* NULL directive, see § 6.10.7 */
1363                 break;
1364         default:
1365                 parse_error("invalid preprocessor directive");
1366                 eat_until_newline();
1367                 break;
1368         }
1369 }
1370
1371 #define MAYBE_PROLOG                                       \
1372                         next_char();                                   \
1373                         while(1) {                                     \
1374                                 switch(c) {
1375
1376 #define MAYBE(ch, set_type)                                \
1377                                 case ch:                                   \
1378                                         next_char();                           \
1379                                         lexer_token.type = set_type;           \
1380                                         return;
1381
1382 #define ELSE_CODE(code)                                    \
1383                                 default:                                   \
1384                                         code                                   \
1385                                 }                                          \
1386                         } /* end of while(1) */                        \
1387                         break;
1388
1389 #define ELSE(set_type)                                     \
1390                 ELSE_CODE(                                         \
1391                         lexer_token.type = set_type;                   \
1392                         return;                                        \
1393                 )
1394
1395 void lexer_next_preprocessing_token(void)
1396 {
1397         while(1) {
1398                 switch(c) {
1399                 case ' ':
1400                 case '\t':
1401                         next_char();
1402                         break;
1403
1404                 MATCH_NEWLINE(
1405                         lexer_token.type = '\n';
1406                         return;
1407                 )
1408
1409                 SYMBOL_CHARS
1410                         parse_symbol();
1411                         /* might be a wide string ( L"string" ) */
1412                         if(lexer_token.type == T_IDENTIFIER &&
1413                             lexer_token.v.symbol == symbol_L) {
1414                             if(c == '"') {
1415                                         parse_wide_string_literal();
1416                                 } else if(c == '\'') {
1417                                         parse_wide_character_constant();
1418                                 }
1419                         }
1420                         return;
1421
1422                 DIGITS
1423                         parse_number();
1424                         return;
1425
1426                 case '"':
1427                         parse_string_literal();
1428                         return;
1429
1430                 case '\'':
1431                         parse_character_constant();
1432                         return;
1433
1434                 case '.':
1435                         MAYBE_PROLOG
1436                                 case '0':
1437                                 case '1':
1438                                 case '2':
1439                                 case '3':
1440                                 case '4':
1441                                 case '5':
1442                                 case '6':
1443                                 case '7':
1444                                 case '8':
1445                                 case '9':
1446                                         put_back(c);
1447                                         c = '.';
1448                                         parse_number_dec();
1449                                         return;
1450
1451                                 case '.':
1452                                         MAYBE_PROLOG
1453                                         MAYBE('.', T_DOTDOTDOT)
1454                                         ELSE_CODE(
1455                                                 put_back(c);
1456                                                 c = '.';
1457                                                 lexer_token.type = '.';
1458                                                 return;
1459                                         )
1460                         ELSE('.')
1461                 case '&':
1462                         MAYBE_PROLOG
1463                         MAYBE('&', T_ANDAND)
1464                         MAYBE('=', T_ANDEQUAL)
1465                         ELSE('&')
1466                 case '*':
1467                         MAYBE_PROLOG
1468                         MAYBE('=', T_ASTERISKEQUAL)
1469                         ELSE('*')
1470                 case '+':
1471                         MAYBE_PROLOG
1472                         MAYBE('+', T_PLUSPLUS)
1473                         MAYBE('=', T_PLUSEQUAL)
1474                         ELSE('+')
1475                 case '-':
1476                         MAYBE_PROLOG
1477                         MAYBE('>', T_MINUSGREATER)
1478                         MAYBE('-', T_MINUSMINUS)
1479                         MAYBE('=', T_MINUSEQUAL)
1480                         ELSE('-')
1481                 case '!':
1482                         MAYBE_PROLOG
1483                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1484                         ELSE('!')
1485                 case '/':
1486                         MAYBE_PROLOG
1487                         MAYBE('=', T_SLASHEQUAL)
1488                                 case '*':
1489                                         next_char();
1490                                         skip_multiline_comment();
1491                                         lexer_next_preprocessing_token();
1492                                         return;
1493                                 case '/':
1494                                         next_char();
1495                                         skip_line_comment();
1496                                         lexer_next_preprocessing_token();
1497                                         return;
1498                         ELSE('/')
1499                 case '%':
1500                         MAYBE_PROLOG
1501                         MAYBE('>', '}')
1502                         MAYBE('=', T_PERCENTEQUAL)
1503                                 case ':':
1504                                         MAYBE_PROLOG
1505                                                 case '%':
1506                                                         MAYBE_PROLOG
1507                                                         MAYBE(':', T_HASHHASH)
1508                                                         ELSE_CODE(
1509                                                                 put_back(c);
1510                                                                 c = '%';
1511                                                                 lexer_token.type = '#';
1512                                                                 return;
1513                                                         )
1514                                         ELSE('#')
1515                         ELSE('%')
1516                 case '<':
1517                         MAYBE_PROLOG
1518                         MAYBE(':', '[')
1519                         MAYBE('%', '{')
1520                         MAYBE('=', T_LESSEQUAL)
1521                                 case '<':
1522                                         MAYBE_PROLOG
1523                                         MAYBE('=', T_LESSLESSEQUAL)
1524                                         ELSE(T_LESSLESS)
1525                         ELSE('<')
1526                 case '>':
1527                         MAYBE_PROLOG
1528                         MAYBE('=', T_GREATEREQUAL)
1529                                 case '>':
1530                                         MAYBE_PROLOG
1531                                         MAYBE('=', T_GREATERGREATEREQUAL)
1532                                         ELSE(T_GREATERGREATER)
1533                         ELSE('>')
1534                 case '^':
1535                         MAYBE_PROLOG
1536                         MAYBE('=', T_CARETEQUAL)
1537                         ELSE('^')
1538                 case '|':
1539                         MAYBE_PROLOG
1540                         MAYBE('=', T_PIPEEQUAL)
1541                         MAYBE('|', T_PIPEPIPE)
1542                         ELSE('|')
1543                 case ':':
1544                         MAYBE_PROLOG
1545                         MAYBE('>', ']')
1546                         ELSE(':')
1547                 case '=':
1548                         MAYBE_PROLOG
1549                         MAYBE('=', T_EQUALEQUAL)
1550                         ELSE('=')
1551                 case '#':
1552                         MAYBE_PROLOG
1553                         MAYBE('#', T_HASHHASH)
1554                         ELSE('#')
1555
1556                 case '?':
1557                 case '[':
1558                 case ']':
1559                 case '(':
1560                 case ')':
1561                 case '{':
1562                 case '}':
1563                 case '~':
1564                 case ';':
1565                 case ',':
1566                 case '\\':
1567                         lexer_token.type = c;
1568                         next_char();
1569                         return;
1570
1571                 case EOF:
1572                         lexer_token.type = T_EOF;
1573                         return;
1574
1575                 default:
1576                         next_char();
1577                         errorf(&lexer_token.source_position, "unknown character '%c' found\n", c);
1578                         lexer_token.type = T_ERROR;
1579                         return;
1580                 }
1581         }
1582 }
1583
1584 void lexer_next_token(void)
1585 {
1586         lexer_next_preprocessing_token();
1587         if(lexer_token.type != '\n')
1588                 return;
1589
1590 newline_found:
1591         do {
1592                 lexer_next_preprocessing_token();
1593         } while(lexer_token.type == '\n');
1594
1595         if(lexer_token.type == '#') {
1596                 parse_preprocessor_directive();
1597                 goto newline_found;
1598         }
1599 }
1600
1601 void init_lexer(void)
1602 {
1603         strset_init(&stringset);
1604         symbol_L = symbol_table_insert("L");
1605 }
1606
1607 void lexer_open_stream(FILE *stream, const char *input_name)
1608 {
1609         input                                  = stream;
1610         lexer_token.source_position.linenr     = 0;
1611         lexer_token.source_position.input_name = input_name;
1612
1613         bufpos = NULL;
1614         bufend = NULL;
1615
1616         /* place a virtual \n at the beginning so the lexer knows that we're
1617          * at the beginning of a line */
1618         c = '\n';
1619 }
1620
1621 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1622 {
1623         input                                  = NULL;
1624         lexer_token.source_position.linenr     = 0;
1625         lexer_token.source_position.input_name = input_name;
1626
1627         bufpos = buffer;
1628         bufend = buffer + len;
1629
1630         /* place a virtual \n at the beginning so the lexer knows that we're
1631          * at the beginning of a line */
1632         c = '\n';
1633 }
1634
1635 void exit_lexer(void)
1636 {
1637         strset_destroy(&stringset);
1638 }
1639
1640 static __attribute__((unused))
1641 void dbg_pos(const source_position_t source_position)
1642 {
1643         fprintf(stdout, "%s:%u\n", source_position.input_name,
1644                 source_position.linenr);
1645         fflush(stdout);
1646 }