nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 //#define DEBUG_CHARS
  44 #define MAX_PUTBACK 3
  45
  46 #ifdef _WIN32
  47 /* No strtold on windows and no replacement yet */
  48 #define strtold(s, e) strtod(s, e)
  49 #endif
  50
  51 static int         c;
  52 token_t            lexer_token;
  53 symbol_t          *symbol_L;
  54 static FILE       *input;
  55 static char        buf[1024 + MAX_PUTBACK];
  56 static const char *bufend;
  57 static const char *bufpos;
  58 static strset_t    stringset;
  59
  60 /**
  61  * Prints a parse error message at the current token.
  62  *
  63  * @param msg   the error message
  64  */
  65 static void parse_error(const char *msg)
  66 {
  67         errorf(lexer_token.source_position,  "%s", msg);
  68 }
  69
  70 static inline void next_real_char(void)
  71 {
  72         assert(bufpos <= bufend);
  73         if (bufpos >= bufend) {
  74                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  75                                  input);
  76                 if(s == 0) {
  77                         c = EOF;
  78                         return;
  79                 }
  80                 bufpos = buf + MAX_PUTBACK;
  81                 bufend = buf + MAX_PUTBACK + s;
  82         }
  83         c = *bufpos++;
  84 }
  85
  86 /**
  87  * Put a character back into the buffer.
  88  *
  89  * @param pc  the character to put back
  90  */
  91 static inline void put_back(int pc)
  92 {
  93         assert(bufpos > buf);
  94         *(--bufpos - buf + buf) = (char) pc;
  95
  96 #ifdef DEBUG_CHARS
  97         printf("putback '%c'\n", pc);
  98 #endif
  99 }
 100
 101 static inline void next_char(void);
 102
 103 #define MATCH_NEWLINE(code)                   \
 104         case '\r':                                \
 105                 next_char();                          \
 106                 if(c == '\n') {                       \
 107                         next_char();                      \
 108                 }                                     \
 109                 lexer_token.source_position.linenr++; \
 110                 code                                  \
 111         case '\n':                                \
 112                 next_char();                          \
 113                 lexer_token.source_position.linenr++; \
 114                 code
 115
 116 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 117
 118 static void maybe_concat_lines(void)
 119 {
 120         eat('\\');
 121
 122         switch(c) {
 123         MATCH_NEWLINE(return;)
 124
 125         default:
 126                 break;
 127         }
 128
 129         put_back(c);
 130         c = '\\';
 131 }
 132
 133 /**
 134  * Set c to the next input character, ie.
 135  * after expanding trigraphs.
 136  */
 137 static inline void next_char(void)
 138 {
 139         next_real_char();
 140
 141         /* filter trigraphs */
 142         if(UNLIKELY(c == '\\')) {
 143                 maybe_concat_lines();
 144                 goto end_of_next_char;
 145         }
 146
 147         if(LIKELY(c != '?'))
 148                 goto end_of_next_char;
 149
 150         next_real_char();
 151         if(LIKELY(c != '?')) {
 152                 put_back(c);
 153                 c = '?';
 154                 goto end_of_next_char;
 155         }
 156
 157         next_real_char();
 158         switch(c) {
 159         case '=': c = '#'; break;
 160         case '(': c = '['; break;
 161         case '/': c = '\\'; maybe_concat_lines(); break;
 162         case ')': c = ']'; break;
 163         case '\'': c = '^'; break;
 164         case '<': c = '{'; break;
 165         case '!': c = '|'; break;
 166         case '>': c = '}'; break;
 167         case '-': c = '~'; break;
 168         default:
 169                 put_back(c);
 170                 put_back('?');
 171                 c = '?';
 172                 break;
 173         }
 174
 175 end_of_next_char:;
 176 #ifdef DEBUG_CHARS
 177         printf("nchar '%c'\n", c);
 178 #endif
 179 }
 180
 181 #define SYMBOL_CHARS  \
 182         case 'a':         \
 183         case 'b':         \
 184         case 'c':         \
 185         case 'd':         \
 186         case 'e':         \
 187         case 'f':         \
 188         case 'g':         \
 189         case 'h':         \
 190         case 'i':         \
 191         case 'j':         \
 192         case 'k':         \
 193         case 'l':         \
 194         case 'm':         \
 195         case 'n':         \
 196         case 'o':         \
 197         case 'p':         \
 198         case 'q':         \
 199         case 'r':         \
 200         case 's':         \
 201         case 't':         \
 202         case 'u':         \
 203         case 'v':         \
 204         case 'w':         \
 205         case 'x':         \
 206         case 'y':         \
 207         case 'z':         \
 208         case 'A':         \
 209         case 'B':         \
 210         case 'C':         \
 211         case 'D':         \
 212         case 'E':         \
 213         case 'F':         \
 214         case 'G':         \
 215         case 'H':         \
 216         case 'I':         \
 217         case 'J':         \
 218         case 'K':         \
 219         case 'L':         \
 220         case 'M':         \
 221         case 'N':         \
 222         case 'O':         \
 223         case 'P':         \
 224         case 'Q':         \
 225         case 'R':         \
 226         case 'S':         \
 227         case 'T':         \
 228         case 'U':         \
 229         case 'V':         \
 230         case 'W':         \
 231         case 'X':         \
 232         case 'Y':         \
 233         case 'Z':         \
 234         case '_':
 235
 236 #define DIGITS        \
 237         case '0':         \
 238         case '1':         \
 239         case '2':         \
 240         case '3':         \
 241         case '4':         \
 242         case '5':         \
 243         case '6':         \
 244         case '7':         \
 245         case '8':         \
 246         case '9':
 247
 248 /**
 249  * Read a symbol from the input and build
 250  * the lexer_token.
 251  */
 252 static void parse_symbol(void)
 253 {
 254         symbol_t *symbol;
 255         char     *string;
 256
 257         obstack_1grow(&symbol_obstack, (char) c);
 258         next_char();
 259
 260         while(1) {
 261                 switch(c) {
 262                 DIGITS
 263                 SYMBOL_CHARS
 264                         obstack_1grow(&symbol_obstack, (char) c);
 265                         next_char();
 266                         break;
 267
 268                 default:
 269                         goto end_symbol;
 270                 }
 271         }
 272
 273 end_symbol:
 274         obstack_1grow(&symbol_obstack, '\0');
 275
 276         string = obstack_finish(&symbol_obstack);
 277         symbol = symbol_table_insert(string);
 278
 279         lexer_token.type     = symbol->ID;
 280         lexer_token.v.symbol = symbol;
 281
 282         if(symbol->string != string) {
 283                 obstack_free(&symbol_obstack, string);
 284         }
 285 }
 286
 287 static void parse_integer_suffix(bool is_oct_hex)
 288 {
 289         bool is_unsigned  = false;
 290         bool min_long     = false;
 291         bool min_longlong = false;
 292
 293         if(c == 'U' || c == 'u') {
 294                 is_unsigned = true;
 295                 next_char();
 296                 if(c == 'L' || c == 'l') {
 297                         min_long = true;
 298                         next_char();
 299                         if(c == 'L' || c == 'l') {
 300                                 min_longlong = true;
 301                                 next_char();
 302                         }
 303                 }
 304         } else if(c == 'l' || c == 'L') {
 305                 min_long = true;
 306                 next_char();
 307                 if(c == 'l' || c == 'L') {
 308                         min_longlong = true;
 309                         next_char();
 310                         if(c == 'u' || c == 'U') {
 311                                 is_unsigned = true;
 312                                 next_char();
 313                         }
 314                 } else if(c == 'u' || c == 'U') {
 315                         is_unsigned = true;
 316                         next_char();
 317                         lexer_token.datatype = type_unsigned_long;
 318                 }
 319         }
 320
 321         if(!is_unsigned) {
 322                 long long v = lexer_token.v.intvalue;
 323                 if(!min_long) {
 324                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 325                                 lexer_token.datatype = type_int;
 326                                 return;
 327                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 328                                 lexer_token.datatype = type_unsigned_int;
 329                                 return;
 330                         }
 331                 }
 332                 if(!min_longlong) {
 333                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 334                                 lexer_token.datatype = type_long;
 335                                 return;
 336                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 337                                 lexer_token.datatype = type_unsigned_long;
 338                                 return;
 339                         }
 340                 }
 341                 unsigned long long uv = (unsigned long long) v;
 342                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 343                         lexer_token.datatype = type_unsigned_long_long;
 344                         return;
 345                 }
 346
 347                 lexer_token.datatype = type_long_long;
 348         } else {
 349                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 350                 if(!min_long && v <= TARGET_UINT_MAX) {
 351                         lexer_token.datatype = type_unsigned_int;
 352                         return;
 353                 }
 354                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 355                         lexer_token.datatype = type_unsigned_long;
 356                         return;
 357                 }
 358                 lexer_token.datatype = type_unsigned_long_long;
 359         }
 360 }
 361
 362 static void parse_floating_suffix(void)
 363 {
 364         switch(c) {
 365         /* TODO: do something useful with the suffixes... */
 366         case 'f':
 367         case 'F':
 368                 next_char();
 369                 lexer_token.datatype = type_float;
 370                 break;
 371         case 'l':
 372         case 'L':
 373                 next_char();
 374                 lexer_token.datatype = type_long_double;
 375                 break;
 376         default:
 377                 lexer_token.datatype = type_double;
 378                 break;
 379         }
 380 }
 381
 382 /**
 383  * A replacement for strtoull. Only those parts needed for
 384  * our parser are implemented.
 385  */
 386 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 387         unsigned long long v = 0;
 388
 389         switch (base) {
 390         case 16:
 391                 for (;; ++s) {
 392                         /* check for overrun */
 393                         if (v >= 0x1000000000000000ULL)
 394                                 break;
 395                         switch (tolower(*s)) {
 396                         case '0': v <<= 4; break;
 397                         case '1': v <<= 4; v |= 0x1; break;
 398                         case '2': v <<= 4; v |= 0x2; break;
 399                         case '3': v <<= 4; v |= 0x3; break;
 400                         case '4': v <<= 4; v |= 0x4; break;
 401                         case '5': v <<= 4; v |= 0x5; break;
 402                         case '6': v <<= 4; v |= 0x6; break;
 403                         case '7': v <<= 4; v |= 0x7; break;
 404                         case '8': v <<= 4; v |= 0x8; break;
 405                         case '9': v <<= 4; v |= 0x9; break;
 406                         case 'a': v <<= 4; v |= 0xa; break;
 407                         case 'b': v <<= 4; v |= 0xb; break;
 408                         case 'c': v <<= 4; v |= 0xc; break;
 409                         case 'd': v <<= 4; v |= 0xd; break;
 410                         case 'e': v <<= 4; v |= 0xe; break;
 411                         case 'f': v <<= 4; v |= 0xf; break;
 412                         default:
 413                                 goto end;
 414                         }
 415                 }
 416                 break;
 417         case 8:
 418                 for (;; ++s) {
 419                         /* check for overrun */
 420                         if (v >= 0x2000000000000000ULL)
 421                                 break;
 422                         switch (tolower(*s)) {
 423                         case '0': v <<= 3; break;
 424                         case '1': v <<= 3; v |= 1; break;
 425                         case '2': v <<= 3; v |= 2; break;
 426                         case '3': v <<= 3; v |= 3; break;
 427                         case '4': v <<= 3; v |= 4; break;
 428                         case '5': v <<= 3; v |= 5; break;
 429                         case '6': v <<= 3; v |= 6; break;
 430                         case '7': v <<= 3; v |= 7; break;
 431                         default:
 432                                 goto end;
 433                         }
 434                 }
 435                 break;
 436         case 10:
 437                 for (;; ++s) {
 438                         /* check for overrun */
 439                         if (v > 0x1999999999999999ULL)
 440                                 break;
 441                         switch (tolower(*s)) {
 442                         case '0': v *= 10; break;
 443                         case '1': v *= 10; v += 1; break;
 444                         case '2': v *= 10; v += 2; break;
 445                         case '3': v *= 10; v += 3; break;
 446                         case '4': v *= 10; v += 4; break;
 447                         case '5': v *= 10; v += 5; break;
 448                         case '6': v *= 10; v += 6; break;
 449                         case '7': v *= 10; v += 7; break;
 450                         case '8': v *= 10; v += 8; break;
 451                         case '9': v *= 10; v += 9; break;
 452                         default:
 453                                 goto end;
 454                         }
 455                 }
 456                 break;
 457         default:
 458                 assert(0);
 459                 break;
 460         }
 461 end:
 462         *endptr = s;
 463         return v;
 464 }
 465
 466 /**
 467  * Parses a hex number including hex floats and set the
 468  * lexer_token.
 469  */
 470 static void parse_number_hex(void)
 471 {
 472         assert(c == 'x' || c == 'X');
 473         next_char();
 474
 475         while(isxdigit(c)) {
 476                 obstack_1grow(&symbol_obstack, (char) c);
 477                 next_char();
 478         }
 479         obstack_1grow(&symbol_obstack, '\0');
 480         char *string = obstack_finish(&symbol_obstack);
 481
 482         if(c == '.' || c == 'p' || c == 'P') {
 483                 next_char();
 484                 panic("Hex floating point numbers not implemented yet");
 485         }
 486         if(*string == '\0') {
 487                 parse_error("invalid hex number");
 488                 lexer_token.type = T_ERROR;
 489         }
 490
 491         const char *endptr;
 492         lexer_token.type       = T_INTEGER;
 493         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 494         if(*endptr != '\0') {
 495                 parse_error("hex number literal too long");
 496         }
 497
 498         obstack_free(&symbol_obstack, string);
 499         parse_integer_suffix(true);
 500 }
 501
 502 /**
 503  * Returns true if the given char is a octal digit.
 504  *
 505  * @param char  the character to check
 506  */
 507 static inline bool is_octal_digit(int chr)
 508 {
 509         switch(chr) {
 510         case '0':
 511         case '1':
 512         case '2':
 513         case '3':
 514         case '4':
 515         case '5':
 516         case '6':
 517         case '7':
 518                 return true;
 519         default:
 520                 return false;
 521         }
 522 }
 523
 524 /**
 525  * Parses a octal number and set the lexer_token.
 526  */
 527 static void parse_number_oct(void)
 528 {
 529         while(is_octal_digit(c)) {
 530                 obstack_1grow(&symbol_obstack, (char) c);
 531                 next_char();
 532         }
 533         obstack_1grow(&symbol_obstack, '\0');
 534         char *string = obstack_finish(&symbol_obstack);
 535
 536         const char *endptr;
 537         lexer_token.type       = T_INTEGER;
 538         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 539         if(*endptr != '\0') {
 540                 parse_error("octal number literal too long");
 541         }
 542
 543         obstack_free(&symbol_obstack, string);
 544         parse_integer_suffix(true);
 545 }
 546
 547 /**
 548  * Parses a decimal including float number and set the
 549  * lexer_token.
 550  */
 551 static void parse_number_dec(void)
 552 {
 553         bool is_float = false;
 554         while(isdigit(c)) {
 555                 obstack_1grow(&symbol_obstack, (char) c);
 556                 next_char();
 557         }
 558
 559         if(c == '.') {
 560                 obstack_1grow(&symbol_obstack, '.');
 561                 next_char();
 562
 563                 while(isdigit(c)) {
 564                         obstack_1grow(&symbol_obstack, (char) c);
 565                         next_char();
 566                 }
 567                 is_float = true;
 568         }
 569         if(c == 'e' || c == 'E') {
 570                 obstack_1grow(&symbol_obstack, 'e');
 571                 next_char();
 572
 573                 if(c == '-' || c == '+') {
 574                         obstack_1grow(&symbol_obstack, (char) c);
 575                         next_char();
 576                 }
 577
 578                 while(isdigit(c)) {
 579                         obstack_1grow(&symbol_obstack, (char) c);
 580                         next_char();
 581                 }
 582                 is_float = true;
 583         }
 584
 585         obstack_1grow(&symbol_obstack, '\0');
 586         char *string = obstack_finish(&symbol_obstack);
 587
 588         if(is_float) {
 589                 char *endptr;
 590                 lexer_token.type         = T_FLOATINGPOINT;
 591                 lexer_token.v.floatvalue = strtold(string, &endptr);
 592
 593                 if(*endptr != '\0') {
 594                         parse_error("invalid number literal");
 595                 }
 596
 597                 parse_floating_suffix();
 598         } else {
 599                 const char *endptr;
 600                 lexer_token.type       = T_INTEGER;
 601                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 602
 603                 if(*endptr != '\0') {
 604                         parse_error("invalid number literal");
 605                 }
 606
 607                 parse_integer_suffix(false);
 608         }
 609         obstack_free(&symbol_obstack, string);
 610 }
 611
 612 /**
 613  * Parses a number and sets the lexer_token.
 614  */
 615 static void parse_number(void)
 616 {
 617         if (c == '0') {
 618                 next_char();
 619                 switch (c) {
 620                         case 'X':
 621                         case 'x':
 622                                 parse_number_hex();
 623                                 break;
 624                         case '0':
 625                         case '1':
 626                         case '2':
 627                         case '3':
 628                         case '4':
 629                         case '5':
 630                         case '6':
 631                         case '7':
 632                                 parse_number_oct();
 633                                 break;
 634                         case '8':
 635                         case '9':
 636                                 next_char();
 637                                 parse_error("invalid octal number");
 638                                 lexer_token.type = T_ERROR;
 639                                 return;
 640                         case '.':
 641                         case 'e':
 642                         case 'E':
 643                         default:
 644                                 obstack_1grow(&symbol_obstack, '0');
 645                                 parse_number_dec();
 646                                 return;
 647                 }
 648         } else {
 649                 parse_number_dec();
 650         }
 651 }
 652
 653 /**
 654  * Returns the value of a digit.
 655  * The only portable way to do it ...
 656  */
 657 static int digit_value(int digit) {
 658         switch (digit) {
 659         case '0': return 0;
 660         case '1': return 1;
 661         case '2': return 2;
 662         case '3': return 3;
 663         case '4': return 4;
 664         case '5': return 5;
 665         case '6': return 6;
 666         case '7': return 7;
 667         case '8': return 8;
 668         case '9': return 9;
 669         case 'a':
 670         case 'A': return 10;
 671         case 'b':
 672         case 'B': return 11;
 673         case 'c':
 674         case 'C': return 12;
 675         case 'd':
 676         case 'D': return 13;
 677         case 'e':
 678         case 'E': return 14;
 679         case 'f':
 680         case 'F': return 15;
 681         default:
 682                 panic("wrong character given");
 683         }
 684 }
 685
 686 /**
 687  * Parses an octal character sequence.
 688  *
 689  * @param first_digit  the already read first digit
 690  */
 691 static int parse_octal_sequence(const int first_digit)
 692 {
 693         assert(is_octal_digit(first_digit));
 694         int value = digit_value(first_digit);
 695         if (!is_octal_digit(c)) return value;
 696         value = 8 * value + digit_value(c);
 697         next_char();
 698         if (!is_octal_digit(c)) return value;
 699         value = 8 * value + digit_value(c);
 700         next_char();
 701
 702         if(char_is_signed) {
 703                 return (signed char) value;
 704         } else {
 705                 return (unsigned char) value;
 706         }
 707 }
 708
 709 /**
 710  * Parses a hex character sequence.
 711  */
 712 static int parse_hex_sequence(void)
 713 {
 714         int value = 0;
 715         while(isxdigit(c)) {
 716                 value = 16 * value + digit_value(c);
 717                 next_char();
 718         }
 719
 720         if(char_is_signed) {
 721                 return (signed char) value;
 722         } else {
 723                 return (unsigned char) value;
 724         }
 725 }
 726
 727 /**
 728  * Parse an escape sequence.
 729  */
 730 static int parse_escape_sequence(void)
 731 {
 732         eat('\\');
 733
 734         int ec = c;
 735         next_char();
 736
 737         switch(ec) {
 738         case '"':  return '"';
 739         case '\'': return '\'';
 740         case '\\': return '\\';
 741         case '?': return '\?';
 742         case 'a': return '\a';
 743         case 'b': return '\b';
 744         case 'f': return '\f';
 745         case 'n': return '\n';
 746         case 'r': return '\r';
 747         case 't': return '\t';
 748         case 'v': return '\v';
 749         case 'x':
 750                 return parse_hex_sequence();
 751         case '0':
 752         case '1':
 753         case '2':
 754         case '3':
 755         case '4':
 756         case '5':
 757         case '6':
 758         case '7':
 759                 return parse_octal_sequence(ec);
 760         case EOF:
 761                 parse_error("reached end of file while parsing escape sequence");
 762                 return EOF;
 763         default:
 764                 parse_error("unknown escape sequence");
 765                 return EOF;
 766         }
 767 }
 768
 769 /**
 770  * Concatenate two strings.
 771  */
 772 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 773 {
 774         const size_t len1 = s1->size - 1;
 775         const size_t len2 = s2->size - 1;
 776
 777         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 778         memcpy(concat, s1->begin, len1);
 779         memcpy(concat + len1, s2->begin, len2 + 1);
 780
 781 #if 0 /* TODO hash */
 782         const char *result = strset_insert(&stringset, concat);
 783         if(result != concat) {
 784                 obstack_free(&symbol_obstack, concat);
 785         }
 786
 787         return result;
 788 #else
 789         return (string_t){ concat, len1 + len2 + 1 };
 790 #endif
 791 }
 792
 793 /**
 794  * Concatenate a string and a wide string.
 795  */
 796 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 797 {
 798         const size_t len1 = s1->size - 1;
 799         const size_t len2 = s2->size - 1;
 800
 801         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 802         const char *const src = s1->begin;
 803         for (size_t i = 0; i != len1; ++i) {
 804                 concat[i] = src[i];
 805         }
 806         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 807
 808         return (wide_string_t){ concat, len1 + len2 + 1 };
 809 }
 810
 811 /**
 812  * Concatenate two wide strings.
 813  */
 814 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 815 {
 816         const size_t len1 = s1->size - 1;
 817         const size_t len2 = s2->size - 1;
 818
 819         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 820         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 821         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 822
 823         return (wide_string_t){ concat, len1 + len2 + 1 };
 824 }
 825
 826 /**
 827  * Concatenate a wide string and a string.
 828  */
 829 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 830 {
 831         const size_t len1 = s1->size - 1;
 832         const size_t len2 = s2->size - 1;
 833
 834         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 835         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 836         const char *const src = s2->begin;
 837         for (size_t i = 0; i != len2 + 1; ++i) {
 838                 concat[i] = src[i];
 839         }
 840
 841         return (wide_string_t){ concat, len1 + len2 + 1 };
 842 }
 843
 844 /**
 845  * Parse a string literal and set lexer_token.
 846  */
 847 static void parse_string_literal(void)
 848 {
 849         const unsigned start_linenr = lexer_token.source_position.linenr;
 850
 851         eat('"');
 852
 853         int tc;
 854         while(1) {
 855                 switch(c) {
 856                 case '\\':
 857                         tc = parse_escape_sequence();
 858                         obstack_1grow(&symbol_obstack, (char) tc);
 859                         break;
 860
 861                 case EOF: {
 862                         source_position_t source_position;
 863                         source_position.input_name = lexer_token.source_position.input_name;
 864                         source_position.linenr     = start_linenr;
 865                         errorf(source_position, "string has no end");
 866                         lexer_token.type = T_ERROR;
 867                         return;
 868                 }
 869
 870                 case '"':
 871                         next_char();
 872                         goto end_of_string;
 873
 874                 default:
 875                         obstack_1grow(&symbol_obstack, (char) c);
 876                         next_char();
 877                         break;
 878                 }
 879         }
 880
 881 end_of_string:
 882
 883         /* TODO: concatenate multiple strings separated by whitespace... */
 884
 885         /* add finishing 0 to the string */
 886         obstack_1grow(&symbol_obstack, '\0');
 887         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 888         const char *const string = obstack_finish(&symbol_obstack);
 889
 890 #if 0 /* TODO hash */
 891         /* check if there is already a copy of the string */
 892         result = strset_insert(&stringset, string);
 893         if(result != string) {
 894                 obstack_free(&symbol_obstack, string);
 895         }
 896 #else
 897         const char *const result = string;
 898 #endif
 899
 900         lexer_token.type           = T_STRING_LITERAL;
 901         lexer_token.v.string.begin = result;
 902         lexer_token.v.string.size  = size;
 903 }
 904
 905 /**
 906  * Parse a wide character constant and set lexer_token.
 907  */
 908 static void parse_wide_character_constant(void)
 909 {
 910         const unsigned start_linenr = lexer_token.source_position.linenr;
 911
 912         eat('\'');
 913
 914         while(1) {
 915                 switch(c) {
 916                 case '\\': {
 917                         wchar_rep_t tc = parse_escape_sequence();
 918                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 919                         break;
 920                 }
 921
 922                 MATCH_NEWLINE(
 923                         parse_error("newline while parsing character constant");
 924                         break;
 925                 )
 926
 927                 case '\'':
 928                         next_char();
 929                         goto end_of_wide_char_constant;
 930
 931                 case EOF: {
 932                         source_position_t source_position = lexer_token.source_position;
 933                         source_position.linenr = start_linenr;
 934                         errorf(source_position, "EOF while parsing character constant");
 935                         lexer_token.type = T_ERROR;
 936                         return;
 937                 }
 938
 939                 default: {
 940                         wchar_rep_t tc = (wchar_rep_t) c;
 941                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 942                         next_char();
 943                         break;
 944                 }
 945                 }
 946         }
 947
 948 end_of_wide_char_constant:;
 949         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
 950         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
 951
 952         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
 953         lexer_token.v.wide_string.begin = string;
 954         lexer_token.v.wide_string.size  = size;
 955         lexer_token.datatype            = type_wchar_t;
 956 }
 957
 958 /**
 959  * Parse a wide string literal and set lexer_token.
 960  */
 961 static void parse_wide_string_literal(void)
 962 {
 963         const unsigned start_linenr = lexer_token.source_position.linenr;
 964
 965         assert(c == '"');
 966         next_char();
 967
 968         while(1) {
 969                 switch(c) {
 970                 case '\\': {
 971                         wchar_rep_t tc = parse_escape_sequence();
 972                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 973                         break;
 974                 }
 975
 976                 case EOF: {
 977                         source_position_t source_position;
 978                         source_position.input_name = lexer_token.source_position.input_name;
 979                         source_position.linenr     = start_linenr;
 980                         errorf(source_position, "string has no end");
 981                         lexer_token.type = T_ERROR;
 982                         return;
 983                 }
 984
 985                 case '"':
 986                         next_char();
 987                         goto end_of_string;
 988
 989                 default: {
 990                         wchar_rep_t tc = c;
 991                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 992                         next_char();
 993                         break;
 994                 }
 995                 }
 996         }
 997
 998 end_of_string:;
 999
1000         /* TODO: concatenate multiple strings separated by whitespace... */
1001
1002         /* add finishing 0 to the string */
1003         wchar_rep_t nul = L'\0';
1004         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1005         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1006         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1007
1008 #if 0 /* TODO hash */
1009         /* check if there is already a copy of the string */
1010         const wchar_rep_t *const result = strset_insert(&stringset, string);
1011         if(result != string) {
1012                 obstack_free(&symbol_obstack, string);
1013         }
1014 #else
1015         const wchar_rep_t *const result = string;
1016 #endif
1017
1018         lexer_token.type                = T_WIDE_STRING_LITERAL;
1019         lexer_token.v.wide_string.begin = result;
1020         lexer_token.v.wide_string.size  = size;
1021 }
1022
1023 /**
1024  * Parse a character constant and set lexer_token.
1025  */
1026 static void parse_character_constant(void)
1027 {
1028         const unsigned start_linenr = lexer_token.source_position.linenr;
1029
1030         eat('\'');
1031
1032         while(1) {
1033                 switch(c) {
1034                 case '\\': {
1035                         int tc = parse_escape_sequence();
1036                         obstack_1grow(&symbol_obstack, (char) tc);
1037                         break;
1038                 }
1039
1040                 MATCH_NEWLINE(
1041                         parse_error("newline while parsing character constant");
1042                         break;
1043                 )
1044
1045                 case '\'':
1046                         next_char();
1047                         goto end_of_char_constant;
1048
1049                 case EOF: {
1050                         source_position_t source_position;
1051                         source_position.input_name = lexer_token.source_position.input_name;
1052                         source_position.linenr     = start_linenr;
1053                         errorf(source_position, "EOF while parsing character constant");
1054                         lexer_token.type = T_ERROR;
1055                         return;
1056                 }
1057
1058                 default:
1059                         obstack_1grow(&symbol_obstack, (char) c);
1060                         next_char();
1061                         break;
1062
1063                 }
1064         }
1065
1066 end_of_char_constant:;
1067         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1068         const char *const string = obstack_finish(&symbol_obstack);
1069
1070         lexer_token.type           = T_CHARACTER_CONSTANT;
1071         lexer_token.v.string.begin = string;
1072         lexer_token.v.string.size  = size;
1073         lexer_token.datatype       = type_int;
1074 }
1075
1076 /**
1077  * Skip a multiline comment.
1078  */
1079 static void skip_multiline_comment(void)
1080 {
1081         unsigned start_linenr = lexer_token.source_position.linenr;
1082
1083         while(1) {
1084                 switch(c) {
1085                 case '/':
1086                         next_char();
1087                         if (c == '*') {
1088                                 /* TODO: nested comment, warn here */
1089                         }
1090                         break;
1091                 case '*':
1092                         next_char();
1093                         if(c == '/') {
1094                                 next_char();
1095                                 return;
1096                         }
1097                         break;
1098
1099                 MATCH_NEWLINE(break;)
1100
1101                 case EOF: {
1102                         source_position_t source_position;
1103                         source_position.input_name = lexer_token.source_position.input_name;
1104                         source_position.linenr     = start_linenr;
1105                         errorf(source_position, "at end of file while looking for comment end");
1106                         return;
1107                 }
1108
1109                 default:
1110                         next_char();
1111                         break;
1112                 }
1113         }
1114 }
1115
1116 /**
1117  * Skip a single line comment.
1118  */
1119 static void skip_line_comment(void)
1120 {
1121         while(1) {
1122                 switch(c) {
1123                 case EOF:
1124                         return;
1125
1126                 case '\n':
1127                 case '\r':
1128                         return;
1129
1130                 default:
1131                         next_char();
1132                         break;
1133                 }
1134         }
1135 }
1136
1137 /** The current preprocessor token. */
1138 static token_t pp_token;
1139
1140 /**
1141  * Read the next preprocessor token.
1142  */
1143 static inline void next_pp_token(void)
1144 {
1145         lexer_next_preprocessing_token();
1146         pp_token = lexer_token;
1147 }
1148
1149 /**
1150  * Eat all preprocessor tokens until newline.
1151  */
1152 static void eat_until_newline(void)
1153 {
1154         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1155                 next_pp_token();
1156         }
1157 }
1158
1159 /**
1160  * Handle the define directive.
1161  */
1162 static void define_directive(void)
1163 {
1164         lexer_next_preprocessing_token();
1165         if(lexer_token.type != T_IDENTIFIER) {
1166                 parse_error("expected identifier after #define\n");
1167                 eat_until_newline();
1168         }
1169 }
1170
1171 /**
1172  * Handle the ifdef directive.
1173  */
1174 static void ifdef_directive(int is_ifndef)
1175 {
1176         (void) is_ifndef;
1177         lexer_next_preprocessing_token();
1178         //expect_identifier();
1179         //extect_newline();
1180 }
1181
1182 /**
1183  * Handle the endif directive.
1184  */
1185 static void endif_directive(void)
1186 {
1187         //expect_newline();
1188 }
1189
1190 /**
1191  * Parse the line directive.
1192  */
1193 static void parse_line_directive(void)
1194 {
1195         if(pp_token.type != T_INTEGER) {
1196                 parse_error("expected integer");
1197         } else {
1198                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1199                 next_pp_token();
1200         }
1201         if(pp_token.type == T_STRING_LITERAL) {
1202                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1203                 next_pp_token();
1204         }
1205
1206         eat_until_newline();
1207 }
1208
1209 /**
1210  * STDC pragmas.
1211  */
1212 typedef enum {
1213         STDC_UNKNOWN,
1214         STDC_FP_CONTRACT,
1215         STDC_FENV_ACCESS,
1216         STDC_CX_LIMITED_RANGE
1217 } stdc_pragma_kind_t;
1218
1219 /**
1220  * STDC pragma values.
1221  */
1222 typedef enum {
1223         STDC_VALUE_UNKNOWN,
1224         STDC_VALUE_ON,
1225         STDC_VALUE_OFF,
1226         STDC_VALUE_DEFAULT
1227 } stdc_pragma_value_kind_t;
1228
1229 /**
1230  * Parse a pragma directive.
1231  */
1232 static void parse_pragma(void) {
1233         bool unknown_pragma = true;
1234
1235         next_pp_token();
1236         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1237                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1238                 /* a STDC pragma */
1239                 if (c_mode & _C99) {
1240                         next_pp_token();
1241
1242                         switch (pp_token.v.symbol->pp_ID) {
1243                         case TP_FP_CONTRACT:
1244                                 kind = STDC_FP_CONTRACT;
1245                                 break;
1246                         case TP_FENV_ACCESS:
1247                                 kind = STDC_FENV_ACCESS;
1248                                 break;
1249                         case TP_CX_LIMITED_RANGE:
1250                                 kind = STDC_CX_LIMITED_RANGE;
1251                                 break;
1252                         default:
1253                                 break;
1254                         }
1255                         if (kind != STDC_UNKNOWN) {
1256                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1257                                 next_pp_token();
1258                                 switch (pp_token.v.symbol->pp_ID) {
1259                                 case TP_ON:
1260                                         value = STDC_VALUE_ON;
1261                                         break;
1262                                 case TP_OFF:
1263                                         value = STDC_VALUE_OFF;
1264                                         break;
1265                                 case TP_DEFAULT:
1266                                         value = STDC_VALUE_DEFAULT;
1267                                         break;
1268                                 default:
1269                                         break;
1270                                 }
1271                                 if (value != STDC_VALUE_UNKNOWN) {
1272                                         unknown_pragma = false;
1273                                 } else {
1274                                         errorf(pp_token.source_position, "bad STDC pragma argument");
1275                                 }
1276                         }
1277                 }
1278         } else {
1279                 unknown_pragma = true;
1280         }
1281         eat_until_newline();
1282         if (unknown_pragma && warning.unknown_pragmas) {
1283                 warningf(pp_token.source_position, "encountered unknown #pragma");
1284         }
1285 }
1286
1287 /**
1288  * Parse a preprocessor non-null directive.
1289  */
1290 static void parse_preprocessor_identifier(void)
1291 {
1292         assert(pp_token.type == T_IDENTIFIER);
1293         symbol_t *symbol = pp_token.v.symbol;
1294
1295         switch(symbol->pp_ID) {
1296         case TP_include:
1297                 printf("include - enable header name parsing!\n");
1298                 break;
1299         case TP_define:
1300                 define_directive();
1301                 break;
1302         case TP_ifdef:
1303                 ifdef_directive(0);
1304                 break;
1305         case TP_ifndef:
1306                 ifdef_directive(1);
1307                 break;
1308         case TP_endif:
1309                 endif_directive();
1310                 break;
1311         case TP_line:
1312                 next_pp_token();
1313                 parse_line_directive();
1314                 break;
1315         case TP_if:
1316         case TP_else:
1317         case TP_elif:
1318         case TP_undef:
1319         case TP_error:
1320                 /* TODO; output the rest of the line */
1321                 parse_error("#error directive: ");
1322                 break;
1323         case TP_pragma:
1324                 parse_pragma();
1325                 break;
1326         }
1327 }
1328
1329 /**
1330  * Parse a preprocessor directive.
1331  */
1332 static void parse_preprocessor_directive(void)
1333 {
1334         next_pp_token();
1335
1336         switch(pp_token.type) {
1337         case T_IDENTIFIER:
1338                 parse_preprocessor_identifier();
1339                 break;
1340         case T_INTEGER:
1341                 parse_line_directive();
1342                 break;
1343         case '\n':
1344                 /* NULL directive, see § 6.10.7 */
1345                 break;
1346         default:
1347                 parse_error("invalid preprocessor directive");
1348                 eat_until_newline();
1349                 break;
1350         }
1351 }
1352
1353 #define MAYBE_PROLOG                                       \
1354                         next_char();                                   \
1355                         while(1) {                                     \
1356                                 switch(c) {
1357
1358 #define MAYBE(ch, set_type)                                \
1359                                 case ch:                                   \
1360                                         next_char();                           \
1361                                         lexer_token.type = set_type;           \
1362                                         return;
1363
1364 #define ELSE_CODE(code)                                    \
1365                                 default:                                   \
1366                                         code;                                  \
1367                                 }                                          \
1368                         } /* end of while(1) */                        \
1369                         break;
1370
1371 #define ELSE(set_type)                                     \
1372                 ELSE_CODE(                                         \
1373                         lexer_token.type = set_type;                   \
1374                         return;                                        \
1375                 )
1376
1377 void lexer_next_preprocessing_token(void)
1378 {
1379         while(1) {
1380                 switch(c) {
1381                 case ' ':
1382                 case '\t':
1383                         next_char();
1384                         break;
1385
1386                 MATCH_NEWLINE(
1387                         lexer_token.type = '\n';
1388                         return;
1389                 )
1390
1391                 SYMBOL_CHARS
1392                         parse_symbol();
1393                         /* might be a wide string ( L"string" ) */
1394                         if(lexer_token.type == T_IDENTIFIER &&
1395                             lexer_token.v.symbol == symbol_L) {
1396                             if(c == '"') {
1397                                         parse_wide_string_literal();
1398                                 } else if(c == '\'') {
1399                                         parse_wide_character_constant();
1400                                 }
1401                         }
1402                         return;
1403
1404                 DIGITS
1405                         parse_number();
1406                         return;
1407
1408                 case '"':
1409                         parse_string_literal();
1410                         return;
1411
1412                 case '\'':
1413                         parse_character_constant();
1414                         return;
1415
1416                 case '.':
1417                         MAYBE_PROLOG
1418                                 case '0':
1419                                 case '1':
1420                                 case '2':
1421                                 case '3':
1422                                 case '4':
1423                                 case '5':
1424                                 case '6':
1425                                 case '7':
1426                                 case '8':
1427                                 case '9':
1428                                         put_back(c);
1429                                         c = '.';
1430                                         parse_number_dec();
1431                                         return;
1432
1433                                 case '.':
1434                                         MAYBE_PROLOG
1435                                         MAYBE('.', T_DOTDOTDOT)
1436                                         ELSE_CODE(
1437                                                 put_back(c);
1438                                                 c = '.';
1439                                                 lexer_token.type = '.';
1440                                                 return;
1441                                         )
1442                         ELSE('.')
1443                 case '&':
1444                         MAYBE_PROLOG
1445                         MAYBE('&', T_ANDAND)
1446                         MAYBE('=', T_ANDEQUAL)
1447                         ELSE('&')
1448                 case '*':
1449                         MAYBE_PROLOG
1450                         MAYBE('=', T_ASTERISKEQUAL)
1451                         ELSE('*')
1452                 case '+':
1453                         MAYBE_PROLOG
1454                         MAYBE('+', T_PLUSPLUS)
1455                         MAYBE('=', T_PLUSEQUAL)
1456                         ELSE('+')
1457                 case '-':
1458                         MAYBE_PROLOG
1459                         MAYBE('>', T_MINUSGREATER)
1460                         MAYBE('-', T_MINUSMINUS)
1461                         MAYBE('=', T_MINUSEQUAL)
1462                         ELSE('-')
1463                 case '!':
1464                         MAYBE_PROLOG
1465                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1466                         ELSE('!')
1467                 case '/':
1468                         MAYBE_PROLOG
1469                         MAYBE('=', T_SLASHEQUAL)
1470                                 case '*':
1471                                         next_char();
1472                                         skip_multiline_comment();
1473                                         lexer_next_preprocessing_token();
1474                                         return;
1475                                 case '/':
1476                                         next_char();
1477                                         skip_line_comment();
1478                                         lexer_next_preprocessing_token();
1479                                         return;
1480                         ELSE('/')
1481                 case '%':
1482                         MAYBE_PROLOG
1483                         MAYBE('>', '}')
1484                         MAYBE('=', T_PERCENTEQUAL)
1485                                 case ':':
1486                                         MAYBE_PROLOG
1487                                                 case '%':
1488                                                         MAYBE_PROLOG
1489                                                         MAYBE(':', T_HASHHASH)
1490                                                         ELSE_CODE(
1491                                                                 put_back(c);
1492                                                                 c = '%';
1493                                                                 lexer_token.type = '#';
1494                                                                 return;
1495                                                         )
1496                                         ELSE('#')
1497                         ELSE('%')
1498                 case '<':
1499                         MAYBE_PROLOG
1500                         MAYBE(':', '[')
1501                         MAYBE('%', '{')
1502                         MAYBE('=', T_LESSEQUAL)
1503                                 case '<':
1504                                         MAYBE_PROLOG
1505                                         MAYBE('=', T_LESSLESSEQUAL)
1506                                         ELSE(T_LESSLESS)
1507                         ELSE('<')
1508                 case '>':
1509                         MAYBE_PROLOG
1510                         MAYBE('=', T_GREATEREQUAL)
1511                                 case '>':
1512                                         MAYBE_PROLOG
1513                                         MAYBE('=', T_GREATERGREATEREQUAL)
1514                                         ELSE(T_GREATERGREATER)
1515                         ELSE('>')
1516                 case '^':
1517                         MAYBE_PROLOG
1518                         MAYBE('=', T_CARETEQUAL)
1519                         ELSE('^')
1520                 case '|':
1521                         MAYBE_PROLOG
1522                         MAYBE('=', T_PIPEEQUAL)
1523                         MAYBE('|', T_PIPEPIPE)
1524                         ELSE('|')
1525                 case ':':
1526                         MAYBE_PROLOG
1527                         MAYBE('>', ']')
1528                         ELSE(':')
1529                 case '=':
1530                         MAYBE_PROLOG
1531                         MAYBE('=', T_EQUALEQUAL)
1532                         ELSE('=')
1533                 case '#':
1534                         MAYBE_PROLOG
1535                         MAYBE('#', T_HASHHASH)
1536                         ELSE('#')
1537
1538                 case '?':
1539                 case '[':
1540                 case ']':
1541                 case '(':
1542                 case ')':
1543                 case '{':
1544                 case '}':
1545                 case '~':
1546                 case ';':
1547                 case ',':
1548                 case '\\':
1549                         lexer_token.type = c;
1550                         next_char();
1551                         return;
1552
1553                 case EOF:
1554                         lexer_token.type = T_EOF;
1555                         return;
1556
1557                 default:
1558                         next_char();
1559                         errorf(lexer_token.source_position, "unknown character '%c' found\n", c);
1560                         lexer_token.type = T_ERROR;
1561                         return;
1562                 }
1563         }
1564 }
1565
1566 void lexer_next_token(void)
1567 {
1568         lexer_next_preprocessing_token();
1569         if(lexer_token.type != '\n')
1570                 return;
1571
1572 newline_found:
1573         do {
1574                 lexer_next_preprocessing_token();
1575         } while(lexer_token.type == '\n');
1576
1577         if(lexer_token.type == '#') {
1578                 parse_preprocessor_directive();
1579                 goto newline_found;
1580         }
1581 }
1582
1583 void init_lexer(void)
1584 {
1585         strset_init(&stringset);
1586 }
1587
1588 void lexer_open_stream(FILE *stream, const char *input_name)
1589 {
1590         input                                  = stream;
1591         lexer_token.source_position.linenr     = 0;
1592         lexer_token.source_position.input_name = input_name;
1593
1594         symbol_L = symbol_table_insert("L");
1595         bufpos = NULL;
1596         bufend = NULL;
1597
1598         /* place a virtual \n at the beginning so the lexer knows that we're
1599          * at the beginning of a line */
1600         c = '\n';
1601 }
1602
1603 void exit_lexer(void)
1604 {
1605         strset_destroy(&stringset);
1606 }
1607
1608 static __attribute__((unused))
1609 void dbg_pos(const source_position_t source_position)
1610 {
1611         fprintf(stdout, "%s:%u\n", source_position.input_name,
1612                 source_position.linenr);
1613         fflush(stdout);
1614 }