nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 //#define DEBUG_CHARS
  44 #define MAX_PUTBACK 3
  45 #define BUF_SIZE    1024
  46
  47 #if defined(_WIN32) || defined(__CYGWIN__)
  48 /* No strtold on windows and no replacement yet */
  49 #define strtold(s, e) strtod(s, e)
  50 #endif
  51
  52 typedef unsigned int utf32;
  53
  54 static utf32        c;
  55 token_t             lexer_token;
  56 symbol_t           *symbol_L;
  57 static FILE        *input;
  58 static utf32        buf[BUF_SIZE + MAX_PUTBACK];
  59 static const utf32 *bufend;
  60 static const utf32 *bufpos;
  61 static strset_t     stringset;
  62 bool                allow_dollar_in_symbol = true;
  63
  64 /**
  65  * Prints a parse error message at the current token.
  66  *
  67  * @param msg   the error message
  68  */
  69 static void parse_error(const char *msg)
  70 {
  71         errorf(&lexer_token.source_position, "%s", msg);
  72 }
  73
  74 /**
  75  * Prints an internal error message at the current token.
  76  *
  77  * @param msg   the error message
  78  */
  79 static NORETURN internal_error(const char *msg)
  80 {
  81         internal_errorf(&lexer_token.source_position, "%s", msg);
  82 }
  83
  84 static size_t read_block(unsigned char *const read_buf, size_t const n)
  85 {
  86         size_t const s = fread(read_buf, 1, n, input);
  87         if (s == 0) {
  88                 if (ferror(input))
  89                         parse_error("read from input failed");
  90                 buf[MAX_PUTBACK] = EOF;
  91                 bufpos           = buf + MAX_PUTBACK;
  92                 bufend           = buf + MAX_PUTBACK + 1;
  93         }
  94         return s;
  95 }
  96
  97 static void decode_iso_8859_1(void)
  98 {
  99         unsigned char read_buf[BUF_SIZE];
 100         size_t const s = read_block(read_buf, sizeof(read_buf));
 101         if (s == 0)
 102                 return;
 103
 104         unsigned char const *src = read_buf;
 105         unsigned char const *end = read_buf + s;
 106         utf32               *dst = buf + MAX_PUTBACK;
 107         while (src != end)
 108                 *dst++ = *src++;
 109
 110         bufpos = buf + MAX_PUTBACK;
 111         bufend = dst;
 112 }
 113
 114 static void decode_iso_8859_15(void)
 115 {
 116         unsigned char read_buf[BUF_SIZE];
 117         size_t const s = read_block(read_buf, sizeof(read_buf));
 118         if (s == 0)
 119                 return;
 120
 121         unsigned char const *src = read_buf;
 122         unsigned char const *end = read_buf + s;
 123         utf32               *dst = buf + MAX_PUTBACK;
 124         while (src != end) {
 125                 utf32 tc = *src++;
 126                 switch (tc) {
 127                         case 0xA4: tc = 0x20AC; break; // €
 128                         case 0xA6: tc = 0x0160; break; // Š
 129                         case 0xA8: tc = 0x0161; break; // š
 130                         case 0xB4: tc = 0x017D; break; // Ž
 131                         case 0xB8: tc = 0x017E; break; // ž
 132                         case 0xBC: tc = 0x0152; break; // Œ
 133                         case 0xBD: tc = 0x0153; break; // œ
 134                         case 0xBE: tc = 0x0178; break; // Ÿ
 135                 }
 136                 *dst++ = tc;
 137         }
 138
 139         bufpos = buf + MAX_PUTBACK;
 140         bufend = dst;
 141 }
 142
 143 static void decode_utf8(void)
 144 {
 145         static utf32  part_decoded_min_code;
 146         static utf32  part_decoded_char;
 147         static size_t part_decoded_rest_len;
 148
 149         do {
 150                 unsigned char read_buf[BUF_SIZE];
 151                 size_t const s = read_block(read_buf, sizeof(read_buf));
 152                 if (s == 0) {
 153                         if (part_decoded_rest_len > 0)
 154                                 parse_error("incomplete input char at end of input");
 155                         return;
 156                 }
 157
 158                 unsigned char const *src = read_buf;
 159                 unsigned char const *end = read_buf + s;
 160                 utf32               *dst = buf + MAX_PUTBACK;
 161                 utf32                decoded;
 162                 utf32                min_code;
 163
 164                 if (part_decoded_rest_len != 0) {
 165                         min_code              = part_decoded_min_code;
 166                         decoded               = part_decoded_char;
 167                         size_t const rest_len = part_decoded_rest_len;
 168                         part_decoded_rest_len = 0;
 169                         switch (rest_len) {
 170                                 case 4:  goto realign;
 171                                 case 3:  goto three_more;
 172                                 case 2:  goto two_more;
 173                                 default: goto one_more;
 174                         }
 175                 }
 176
 177                 while (src != end) {
 178                         if ((*src & 0x80) == 0) {
 179                                 decoded = *src++;
 180                         } else if ((*src & 0xE0) == 0xC0) {
 181                                 min_code = 0x80;
 182                                 decoded  = *src++ & 0x1F;
 183 one_more:
 184                                 if (src == end) {
 185                                         part_decoded_min_code = min_code;
 186                                         part_decoded_char     = decoded;
 187                                         part_decoded_rest_len = 1;
 188                                         break;
 189                                 }
 190                                 if ((*src & 0xC0) == 0x80) {
 191                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 192                                 } else {
 193                                         goto invalid_char;
 194                                 }
 195                                 if (decoded < min_code                      ||
 196                                                 decoded > 0x10FFFF                      ||
 197                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 198                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 199                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 200                                         parse_error("invalid byte sequence in input");
 201                                 }
 202                         } else if ((*src & 0xF0) == 0xE0) {
 203                                 min_code = 0x800;
 204                                 decoded  = *src++ & 0x0F;
 205 two_more:
 206                                 if (src == end) {
 207                                         part_decoded_min_code = min_code;
 208                                         part_decoded_char     = decoded;
 209                                         part_decoded_rest_len = 2;
 210                                         break;
 211                                 }
 212                                 if ((*src & 0xC0) == 0x80) {
 213                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 214                                 } else {
 215                                         goto invalid_char;
 216                                 }
 217                                 goto one_more;
 218                         } else if ((*src & 0xF8) == 0xF0) {
 219                                 min_code = 0x10000;
 220                                 decoded  = *src++ & 0x07;
 221 three_more:
 222                                 if (src == end) {
 223                                         part_decoded_min_code = min_code;
 224                                         part_decoded_char     = decoded;
 225                                         part_decoded_rest_len = 3;
 226                                         break;
 227                                 }
 228                                 if ((*src & 0xC0) == 0x80) {
 229                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 230                                 } else {
 231                                         goto invalid_char;
 232                                 }
 233                                 goto two_more;
 234                         } else {
 235 invalid_char:
 236                                 parse_error("invalid byte sequence in input");
 237 realign:
 238                                 do {
 239                                         ++src;
 240                                         if (src == end) {
 241                                                 part_decoded_rest_len = 4;
 242                                                 break;
 243                                         }
 244                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 245                                 continue;
 246                         }
 247                         *dst++ = decoded;
 248                 }
 249
 250                 bufpos = buf + MAX_PUTBACK;
 251                 bufend = dst;
 252         } while (bufpos == bufend);
 253 }
 254
 255 typedef void (*decoder_t)(void);
 256
 257 static decoder_t decoder = decode_utf8;
 258
 259 typedef struct named_decoder_t {
 260         char const *name;
 261         decoder_t   decoder;
 262 } named_decoder_t;
 263
 264 static named_decoder_t const decoders[] = {
 265         { "CP819",           decode_iso_8859_1  }, // offical alias
 266         { "IBM819",          decode_iso_8859_1  }, // offical alias
 267         { "ISO-8859-1",      decode_iso_8859_1  }, // offical alias
 268         { "ISO-8859-15",     decode_iso_8859_15 }, // offical name
 269         { "ISO8859-1",       decode_iso_8859_1  },
 270         { "ISO8859-15",      decode_iso_8859_15 },
 271         { "ISO_8859-1",      decode_iso_8859_1  }, // offical alias
 272         { "ISO_8859-15",     decode_iso_8859_15 }, // offical alias
 273         { "ISO_8859-1:1987", decode_iso_8859_1  }, // offical name
 274         { "Latin-9",         decode_iso_8859_15 }, // offical alias
 275         { "UTF-8",           decode_utf8        }, // offical name
 276         { "csISOLatin1",     decode_iso_8859_1  }, // offical alias
 277         { "iso-ir-100",      decode_iso_8859_1  }, // offical alias
 278         { "l1",              decode_iso_8859_1  }, // offical alias
 279         { "latin1",          decode_iso_8859_1  }, // offical alias
 280
 281         { NULL,              NULL               }
 282 };
 283
 284 void select_input_encoding(char const* const encoding)
 285 {
 286         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 287                 if (strcasecmp(encoding, i->name) != 0)
 288                         continue;
 289                 decoder = i->decoder;
 290                 return;
 291         }
 292         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 293 }
 294
 295 static inline void next_real_char(void)
 296 {
 297         assert(bufpos <= bufend);
 298         if (bufpos >= bufend) {
 299                 if (input == NULL) {
 300                         c = EOF;
 301                         return;
 302                 }
 303                 decoder();
 304         }
 305         c = *bufpos++;
 306 }
 307
 308 /**
 309  * Put a character back into the buffer.
 310  *
 311  * @param pc  the character to put back
 312  */
 313 static inline void put_back(utf32 const pc)
 314 {
 315         assert(bufpos > buf);
 316         *(--bufpos - buf + buf) = pc;
 317
 318 #ifdef DEBUG_CHARS
 319         printf("putback '%lc'\n", pc);
 320 #endif
 321 }
 322
 323 static inline void next_char(void);
 324
 325 #define MATCH_NEWLINE(code)                   \
 326         case '\r':                                \
 327                 next_char();                          \
 328                 if(c == '\n') {                       \
 329                         next_char();                      \
 330                 }                                     \
 331                 lexer_token.source_position.linenr++; \
 332                 code                                  \
 333         case '\n':                                \
 334                 next_char();                          \
 335                 lexer_token.source_position.linenr++; \
 336                 code
 337
 338 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 339
 340 static void maybe_concat_lines(void)
 341 {
 342         eat('\\');
 343
 344         switch(c) {
 345         MATCH_NEWLINE(return;)
 346
 347         default:
 348                 break;
 349         }
 350
 351         put_back(c);
 352         c = '\\';
 353 }
 354
 355 /**
 356  * Set c to the next input character, ie.
 357  * after expanding trigraphs.
 358  */
 359 static inline void next_char(void)
 360 {
 361         next_real_char();
 362
 363         /* filter trigraphs */
 364         if(UNLIKELY(c == '\\')) {
 365                 maybe_concat_lines();
 366                 goto end_of_next_char;
 367         }
 368
 369         if(LIKELY(c != '?'))
 370                 goto end_of_next_char;
 371
 372         next_real_char();
 373         if(LIKELY(c != '?')) {
 374                 put_back(c);
 375                 c = '?';
 376                 goto end_of_next_char;
 377         }
 378
 379         next_real_char();
 380         switch(c) {
 381         case '=': c = '#'; break;
 382         case '(': c = '['; break;
 383         case '/': c = '\\'; maybe_concat_lines(); break;
 384         case ')': c = ']'; break;
 385         case '\'': c = '^'; break;
 386         case '<': c = '{'; break;
 387         case '!': c = '|'; break;
 388         case '>': c = '}'; break;
 389         case '-': c = '~'; break;
 390         default:
 391                 put_back(c);
 392                 put_back('?');
 393                 c = '?';
 394                 break;
 395         }
 396
 397 end_of_next_char:;
 398 #ifdef DEBUG_CHARS
 399         printf("nchar '%c'\n", c);
 400 #endif
 401 }
 402
 403 #define SYMBOL_CHARS  \
 404         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 405         case 'a':         \
 406         case 'b':         \
 407         case 'c':         \
 408         case 'd':         \
 409         case 'e':         \
 410         case 'f':         \
 411         case 'g':         \
 412         case 'h':         \
 413         case 'i':         \
 414         case 'j':         \
 415         case 'k':         \
 416         case 'l':         \
 417         case 'm':         \
 418         case 'n':         \
 419         case 'o':         \
 420         case 'p':         \
 421         case 'q':         \
 422         case 'r':         \
 423         case 's':         \
 424         case 't':         \
 425         case 'u':         \
 426         case 'v':         \
 427         case 'w':         \
 428         case 'x':         \
 429         case 'y':         \
 430         case 'z':         \
 431         case 'A':         \
 432         case 'B':         \
 433         case 'C':         \
 434         case 'D':         \
 435         case 'E':         \
 436         case 'F':         \
 437         case 'G':         \
 438         case 'H':         \
 439         case 'I':         \
 440         case 'J':         \
 441         case 'K':         \
 442         case 'L':         \
 443         case 'M':         \
 444         case 'N':         \
 445         case 'O':         \
 446         case 'P':         \
 447         case 'Q':         \
 448         case 'R':         \
 449         case 'S':         \
 450         case 'T':         \
 451         case 'U':         \
 452         case 'V':         \
 453         case 'W':         \
 454         case 'X':         \
 455         case 'Y':         \
 456         case 'Z':         \
 457         case '_':
 458
 459 #define DIGITS        \
 460         case '0':         \
 461         case '1':         \
 462         case '2':         \
 463         case '3':         \
 464         case '4':         \
 465         case '5':         \
 466         case '6':         \
 467         case '7':         \
 468         case '8':         \
 469         case '9':
 470
 471 /**
 472  * Read a symbol from the input and build
 473  * the lexer_token.
 474  */
 475 static void parse_symbol(void)
 476 {
 477         symbol_t *symbol;
 478         char     *string;
 479
 480         obstack_1grow(&symbol_obstack, (char) c);
 481         next_char();
 482
 483         while(1) {
 484                 switch(c) {
 485                 DIGITS
 486                 SYMBOL_CHARS
 487                         obstack_1grow(&symbol_obstack, (char) c);
 488                         next_char();
 489                         break;
 490
 491                 default:
 492 dollar_sign:
 493                         goto end_symbol;
 494                 }
 495         }
 496
 497 end_symbol:
 498         obstack_1grow(&symbol_obstack, '\0');
 499
 500         string = obstack_finish(&symbol_obstack);
 501         symbol = symbol_table_insert(string);
 502
 503         lexer_token.type     = symbol->ID;
 504         lexer_token.v.symbol = symbol;
 505
 506         if(symbol->string != string) {
 507                 obstack_free(&symbol_obstack, string);
 508         }
 509 }
 510
 511 static void parse_integer_suffix(bool is_oct_hex)
 512 {
 513         bool is_unsigned     = false;
 514         bool min_long        = false;
 515         bool min_longlong    = false;
 516         bool not_traditional = false;
 517         int  pos             = 0;
 518         char suffix[4];
 519
 520         if (c == 'U' || c == 'u') {
 521                 not_traditional = true;
 522                 suffix[pos++]   = toupper(c);
 523                 is_unsigned     = true;
 524                 next_char();
 525                 if (c == 'L' || c == 'l') {
 526                         suffix[pos++] = toupper(c);
 527                         min_long = true;
 528                         next_char();
 529                         if (c == 'L' || c == 'l') {
 530                                 suffix[pos++] = toupper(c);
 531                                 min_longlong = true;
 532                                 next_char();
 533                         }
 534                 }
 535         } else if (c == 'l' || c == 'L') {
 536                 suffix[pos++] = toupper(c);
 537                 min_long = true;
 538                 next_char();
 539                 if (c == 'l' || c == 'L') {
 540                         not_traditional = true;
 541                         suffix[pos++]   = toupper(c);
 542                         min_longlong    = true;
 543                         next_char();
 544                         if (c == 'u' || c == 'U') {
 545                                 suffix[pos++] = toupper(c);
 546                                 is_unsigned   = true;
 547                                 next_char();
 548                         }
 549                 } else if (c == 'u' || c == 'U') {
 550                         not_traditional = true;
 551                         suffix[pos++]   = toupper(c);
 552                         is_unsigned     = true;
 553                         next_char();
 554                         lexer_token.datatype = type_unsigned_long;
 555                 }
 556         }
 557
 558         if (warning.traditional && not_traditional) {
 559                 suffix[pos] = '\0';
 560                 warningf(&lexer_token.source_position,
 561                         "traditional C rejects the '%s' suffix", suffix);
 562         }
 563         if (!is_unsigned) {
 564                 long long v = lexer_token.v.intvalue;
 565                 if (!min_long) {
 566                         if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 567                                 lexer_token.datatype = type_int;
 568                                 return;
 569                         } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 570                                 lexer_token.datatype = type_unsigned_int;
 571                                 return;
 572                         }
 573                 }
 574                 if (!min_longlong) {
 575                         if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 576                                 lexer_token.datatype = type_long;
 577                                 return;
 578                         } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) {
 579                                 lexer_token.datatype = type_unsigned_long;
 580                                 return;
 581                         }
 582                 }
 583                 unsigned long long uv = (unsigned long long) v;
 584                 if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 585                         lexer_token.datatype = type_unsigned_long_long;
 586                         return;
 587                 }
 588
 589                 lexer_token.datatype = type_long_long;
 590         } else {
 591                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 592                 if (!min_long && v <= TARGET_UINT_MAX) {
 593                         lexer_token.datatype = type_unsigned_int;
 594                         return;
 595                 }
 596                 if (!min_longlong && v <= TARGET_ULONG_MAX) {
 597                         lexer_token.datatype = type_unsigned_long;
 598                         return;
 599                 }
 600                 lexer_token.datatype = type_unsigned_long_long;
 601         }
 602 }
 603
 604 static void parse_floating_suffix(void)
 605 {
 606         switch(c) {
 607         /* TODO: do something useful with the suffixes... */
 608         case 'f':
 609         case 'F':
 610                 if (warning.traditional) {
 611                         warningf(&lexer_token.source_position,
 612                                 "traditional C rejects the 'F' suffix");
 613                 }
 614                 next_char();
 615                 lexer_token.datatype = type_float;
 616                 break;
 617         case 'l':
 618         case 'L':
 619                 if (warning.traditional) {
 620                         warningf(&lexer_token.source_position,
 621                                 "traditional C rejects the 'F' suffix");
 622                 }
 623                 next_char();
 624                 lexer_token.datatype = type_long_double;
 625                 break;
 626         default:
 627                 lexer_token.datatype = type_double;
 628                 break;
 629         }
 630 }
 631
 632 /**
 633  * A replacement for strtoull. Only those parts needed for
 634  * our parser are implemented.
 635  */
 636 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 637         unsigned long long v = 0;
 638
 639         switch (base) {
 640         case 16:
 641                 for (;; ++s) {
 642                         /* check for overrun */
 643                         if (v >= 0x1000000000000000ULL)
 644                                 break;
 645                         switch (tolower(*s)) {
 646                         case '0': v <<= 4; break;
 647                         case '1': v <<= 4; v |= 0x1; break;
 648                         case '2': v <<= 4; v |= 0x2; break;
 649                         case '3': v <<= 4; v |= 0x3; break;
 650                         case '4': v <<= 4; v |= 0x4; break;
 651                         case '5': v <<= 4; v |= 0x5; break;
 652                         case '6': v <<= 4; v |= 0x6; break;
 653                         case '7': v <<= 4; v |= 0x7; break;
 654                         case '8': v <<= 4; v |= 0x8; break;
 655                         case '9': v <<= 4; v |= 0x9; break;
 656                         case 'a': v <<= 4; v |= 0xa; break;
 657                         case 'b': v <<= 4; v |= 0xb; break;
 658                         case 'c': v <<= 4; v |= 0xc; break;
 659                         case 'd': v <<= 4; v |= 0xd; break;
 660                         case 'e': v <<= 4; v |= 0xe; break;
 661                         case 'f': v <<= 4; v |= 0xf; break;
 662                         default:
 663                                 goto end;
 664                         }
 665                 }
 666                 break;
 667         case 8:
 668                 for (;; ++s) {
 669                         /* check for overrun */
 670                         if (v >= 0x2000000000000000ULL)
 671                                 break;
 672                         switch (tolower(*s)) {
 673                         case '0': v <<= 3; break;
 674                         case '1': v <<= 3; v |= 1; break;
 675                         case '2': v <<= 3; v |= 2; break;
 676                         case '3': v <<= 3; v |= 3; break;
 677                         case '4': v <<= 3; v |= 4; break;
 678                         case '5': v <<= 3; v |= 5; break;
 679                         case '6': v <<= 3; v |= 6; break;
 680                         case '7': v <<= 3; v |= 7; break;
 681                         default:
 682                                 goto end;
 683                         }
 684                 }
 685                 break;
 686         case 10:
 687                 for (;; ++s) {
 688                         /* check for overrun */
 689                         if (v > 0x1999999999999999ULL)
 690                                 break;
 691                         switch (tolower(*s)) {
 692                         case '0': v *= 10; break;
 693                         case '1': v *= 10; v += 1; break;
 694                         case '2': v *= 10; v += 2; break;
 695                         case '3': v *= 10; v += 3; break;
 696                         case '4': v *= 10; v += 4; break;
 697                         case '5': v *= 10; v += 5; break;
 698                         case '6': v *= 10; v += 6; break;
 699                         case '7': v *= 10; v += 7; break;
 700                         case '8': v *= 10; v += 8; break;
 701                         case '9': v *= 10; v += 9; break;
 702                         default:
 703                                 goto end;
 704                         }
 705                 }
 706                 break;
 707         default:
 708                 assert(0);
 709                 break;
 710         }
 711 end:
 712         *endptr = s;
 713         return v;
 714 }
 715
 716 /**
 717  * Parses a hex number including hex floats and set the
 718  * lexer_token.
 719  */
 720 static void parse_number_hex(void)
 721 {
 722         bool is_float = false;
 723         assert(c == 'x' || c == 'X');
 724         next_char();
 725
 726         obstack_1grow(&symbol_obstack, '0');
 727         obstack_1grow(&symbol_obstack, 'x');
 728
 729         while(isxdigit(c)) {
 730                 obstack_1grow(&symbol_obstack, (char) c);
 731                 next_char();
 732         }
 733
 734         if (c == '.') {
 735                 obstack_1grow(&symbol_obstack, (char) c);
 736                 next_char();
 737
 738                 while (isxdigit(c)) {
 739                         obstack_1grow(&symbol_obstack, (char) c);
 740                         next_char();
 741                 }
 742                 is_float = true;
 743         }
 744         if (c == 'p' || c == 'P') {
 745                 obstack_1grow(&symbol_obstack, (char) c);
 746                 next_char();
 747
 748                 if (c == '-' || c == '+') {
 749                         obstack_1grow(&symbol_obstack, (char) c);
 750                         next_char();
 751                 }
 752
 753                 while (isxdigit(c)) {
 754                         obstack_1grow(&symbol_obstack, (char) c);
 755                         next_char();
 756                 }
 757                 is_float = true;
 758         }
 759
 760         obstack_1grow(&symbol_obstack, '\0');
 761         char *string = obstack_finish(&symbol_obstack);
 762         if(*string == '\0') {
 763                 parse_error("invalid hex number");
 764                 lexer_token.type = T_ERROR;
 765                 obstack_free(&symbol_obstack, string);
 766                 return;
 767         }
 768
 769         if (is_float) {
 770                 char *endptr;
 771                 lexer_token.type         = T_FLOATINGPOINT;
 772                 lexer_token.v.floatvalue = strtold(string, &endptr);
 773
 774                 if(*endptr != '\0') {
 775                         parse_error("invalid hex float literal");
 776                 }
 777
 778                 parse_floating_suffix();
 779         } else {
 780                 const char *endptr;
 781                 lexer_token.type       = T_INTEGER;
 782                 lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16);
 783                 if(*endptr != '\0') {
 784                         parse_error("hex number literal too long");
 785                 }
 786                 parse_integer_suffix(true);
 787         }
 788
 789         obstack_free(&symbol_obstack, string);
 790 }
 791
 792 /**
 793  * Returns true if the given char is a octal digit.
 794  *
 795  * @param char  the character to check
 796  */
 797 static inline bool is_octal_digit(utf32 chr)
 798 {
 799         switch(chr) {
 800         case '0':
 801         case '1':
 802         case '2':
 803         case '3':
 804         case '4':
 805         case '5':
 806         case '6':
 807         case '7':
 808                 return true;
 809         default:
 810                 return false;
 811         }
 812 }
 813
 814 /**
 815  * Parses a octal number and set the lexer_token.
 816  */
 817 static void parse_number_oct(void)
 818 {
 819         while(is_octal_digit(c)) {
 820                 obstack_1grow(&symbol_obstack, (char) c);
 821                 next_char();
 822         }
 823         obstack_1grow(&symbol_obstack, '\0');
 824         char *string = obstack_finish(&symbol_obstack);
 825
 826         const char *endptr;
 827         lexer_token.type       = T_INTEGER;
 828         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 829         if(*endptr != '\0') {
 830                 parse_error("octal number literal too long");
 831         }
 832
 833         obstack_free(&symbol_obstack, string);
 834         parse_integer_suffix(true);
 835 }
 836
 837 /**
 838  * Parses a decimal including float number and set the
 839  * lexer_token.
 840  */
 841 static void parse_number_dec(void)
 842 {
 843         bool is_float = false;
 844         while (isdigit(c)) {
 845                 obstack_1grow(&symbol_obstack, (char) c);
 846                 next_char();
 847         }
 848
 849         if (c == '.') {
 850                 obstack_1grow(&symbol_obstack, '.');
 851                 next_char();
 852
 853                 while (isdigit(c)) {
 854                         obstack_1grow(&symbol_obstack, (char) c);
 855                         next_char();
 856                 }
 857                 is_float = true;
 858         }
 859         if(c == 'e' || c == 'E') {
 860                 obstack_1grow(&symbol_obstack, (char) c);
 861                 next_char();
 862
 863                 if(c == '-' || c == '+') {
 864                         obstack_1grow(&symbol_obstack, (char) c);
 865                         next_char();
 866                 }
 867
 868                 while(isdigit(c)) {
 869                         obstack_1grow(&symbol_obstack, (char) c);
 870                         next_char();
 871                 }
 872                 is_float = true;
 873         }
 874
 875         obstack_1grow(&symbol_obstack, '\0');
 876         char *string = obstack_finish(&symbol_obstack);
 877
 878         if(is_float) {
 879                 char *endptr;
 880                 lexer_token.type         = T_FLOATINGPOINT;
 881                 lexer_token.v.floatvalue = strtold(string, &endptr);
 882
 883                 if(*endptr != '\0') {
 884                         parse_error("invalid number literal");
 885                 }
 886
 887                 parse_floating_suffix();
 888         } else {
 889                 const char *endptr;
 890                 lexer_token.type       = T_INTEGER;
 891                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 892
 893                 if(*endptr != '\0') {
 894                         parse_error("invalid number literal");
 895                 }
 896
 897                 parse_integer_suffix(false);
 898         }
 899         obstack_free(&symbol_obstack, string);
 900 }
 901
 902 /**
 903  * Parses a number and sets the lexer_token.
 904  */
 905 static void parse_number(void)
 906 {
 907         if (c == '0') {
 908                 next_char();
 909                 switch (c) {
 910                         case 'X':
 911                         case 'x':
 912                                 parse_number_hex();
 913                                 break;
 914                         case '0':
 915                         case '1':
 916                         case '2':
 917                         case '3':
 918                         case '4':
 919                         case '5':
 920                         case '6':
 921                         case '7':
 922                                 parse_number_oct();
 923                                 break;
 924                         case '8':
 925                         case '9':
 926                                 next_char();
 927                                 parse_error("invalid octal number");
 928                                 lexer_token.type = T_ERROR;
 929                                 return;
 930                         case '.':
 931                         case 'e':
 932                         case 'E':
 933                         default:
 934                                 obstack_1grow(&symbol_obstack, '0');
 935                                 parse_number_dec();
 936                                 return;
 937                 }
 938         } else {
 939                 parse_number_dec();
 940         }
 941 }
 942
 943 /**
 944  * Returns the value of a digit.
 945  * The only portable way to do it ...
 946  */
 947 static int digit_value(utf32 const digit)
 948 {
 949         switch (digit) {
 950         case '0': return 0;
 951         case '1': return 1;
 952         case '2': return 2;
 953         case '3': return 3;
 954         case '4': return 4;
 955         case '5': return 5;
 956         case '6': return 6;
 957         case '7': return 7;
 958         case '8': return 8;
 959         case '9': return 9;
 960         case 'a':
 961         case 'A': return 10;
 962         case 'b':
 963         case 'B': return 11;
 964         case 'c':
 965         case 'C': return 12;
 966         case 'd':
 967         case 'D': return 13;
 968         case 'e':
 969         case 'E': return 14;
 970         case 'f':
 971         case 'F': return 15;
 972         default:
 973                 internal_error("wrong character given");
 974         }
 975 }
 976
 977 /**
 978  * Parses an octal character sequence.
 979  *
 980  * @param first_digit  the already read first digit
 981  */
 982 static utf32 parse_octal_sequence(utf32 const first_digit)
 983 {
 984         assert(is_octal_digit(first_digit));
 985         utf32 value = digit_value(first_digit);
 986         if (!is_octal_digit(c)) return value;
 987         value = 8 * value + digit_value(c);
 988         next_char();
 989         if (!is_octal_digit(c)) return value;
 990         value = 8 * value + digit_value(c);
 991         next_char();
 992         return value;
 993 }
 994
 995 /**
 996  * Parses a hex character sequence.
 997  */
 998 static utf32 parse_hex_sequence(void)
 999 {
1000         utf32 value = 0;
1001         while(isxdigit(c)) {
1002                 value = 16 * value + digit_value(c);
1003                 next_char();
1004         }
1005         return value;
1006 }
1007
1008 /**
1009  * Parse an escape sequence.
1010  */
1011 static utf32 parse_escape_sequence(void)
1012 {
1013         eat('\\');
1014
1015         utf32 const ec = c;
1016         next_char();
1017
1018         switch (ec) {
1019         case '"':  return '"';
1020         case '\'': return '\'';
1021         case '\\': return '\\';
1022         case '?': return '\?';
1023         case 'a': return '\a';
1024         case 'b': return '\b';
1025         case 'f': return '\f';
1026         case 'n': return '\n';
1027         case 'r': return '\r';
1028         case 't': return '\t';
1029         case 'v': return '\v';
1030         case 'x':
1031                 return parse_hex_sequence();
1032         case '0':
1033         case '1':
1034         case '2':
1035         case '3':
1036         case '4':
1037         case '5':
1038         case '6':
1039         case '7':
1040                 return parse_octal_sequence(ec);
1041         case EOF:
1042                 parse_error("reached end of file while parsing escape sequence");
1043                 return EOF;
1044         /* \E is not documented, but handled, by GCC.  It is acceptable according
1045          * to §6.11.4, whereas \e is not. */
1046         case 'E':
1047         case 'e':
1048                 if (c_mode & _GNUC)
1049                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
1050                 /* FALLTHROUGH */
1051         default:
1052                 /* §6.4.4.4:8 footnote 64 */
1053                 parse_error("unknown escape sequence");
1054                 return EOF;
1055         }
1056 }
1057
1058 /**
1059  * Concatenate two strings.
1060  */
1061 string_t concat_strings(const string_t *const s1, const string_t *const s2)
1062 {
1063         const size_t len1 = s1->size - 1;
1064         const size_t len2 = s2->size - 1;
1065
1066         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
1067         memcpy(concat, s1->begin, len1);
1068         memcpy(concat + len1, s2->begin, len2 + 1);
1069
1070         if (warning.traditional) {
1071                 warningf(&lexer_token.source_position,
1072                         "traditional C rejects string constant concatenation");
1073         }
1074 #if 0 /* TODO hash */
1075         const char *result = strset_insert(&stringset, concat);
1076         if(result != concat) {
1077                 obstack_free(&symbol_obstack, concat);
1078         }
1079
1080         return result;
1081 #else
1082         return (string_t){ concat, len1 + len2 + 1 };
1083 #endif
1084 }
1085
1086 /**
1087  * Concatenate a string and a wide string.
1088  */
1089 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
1090 {
1091         const size_t len1 = s1->size - 1;
1092         const size_t len2 = s2->size - 1;
1093
1094         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1095         const char *const src = s1->begin;
1096         for (size_t i = 0; i != len1; ++i) {
1097                 concat[i] = src[i];
1098         }
1099         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
1100         if (warning.traditional) {
1101                 warningf(&lexer_token.source_position,
1102                         "traditional C rejects string constant concatenation");
1103         }
1104
1105         return (wide_string_t){ concat, len1 + len2 + 1 };
1106 }
1107
1108 /**
1109  * Concatenate two wide strings.
1110  */
1111 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
1112 {
1113         const size_t len1 = s1->size - 1;
1114         const size_t len2 = s2->size - 1;
1115
1116         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1117         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
1118         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
1119         if (warning.traditional) {
1120                 warningf(&lexer_token.source_position,
1121                         "traditional C rejects string constant concatenation");
1122         }
1123
1124         return (wide_string_t){ concat, len1 + len2 + 1 };
1125 }
1126
1127 /**
1128  * Concatenate a wide string and a string.
1129  */
1130 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
1131 {
1132         const size_t len1 = s1->size - 1;
1133         const size_t len2 = s2->size - 1;
1134
1135         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1136         memcpy(concat, s1->begin, len1 * sizeof(*concat));
1137         const char  *const src = s2->begin;
1138         wchar_rep_t *const dst = concat + len1;
1139         for (size_t i = 0; i != len2 + 1; ++i) {
1140                 dst[i] = src[i];
1141         }
1142         if (warning.traditional) {
1143                 warningf(&lexer_token.source_position,
1144                         "traditional C rejects string constant concatenation");
1145         }
1146
1147         return (wide_string_t){ concat, len1 + len2 + 1 };
1148 }
1149
1150 static void grow_symbol(utf32 const tc)
1151 {
1152         struct obstack *const o  = &symbol_obstack;
1153         if (tc < 0x80U) {
1154                 obstack_1grow(o, tc);
1155         } else if (tc < 0x800) {
1156                 obstack_1grow(o, 0xC0 | (tc >> 6));
1157                 obstack_1grow(o, 0x80 | (tc & 0x3F));
1158         } else if (tc < 0x10000) {
1159                 obstack_1grow(o, 0xE0 | ( tc >> 12));
1160                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
1161                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
1162         } else {
1163                 obstack_1grow(o, 0xF0 | ( tc >> 18));
1164                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
1165                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
1166                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
1167         }
1168 }
1169
1170 /**
1171  * Parse a string literal and set lexer_token.
1172  */
1173 static void parse_string_literal(void)
1174 {
1175         const unsigned start_linenr = lexer_token.source_position.linenr;
1176
1177         eat('"');
1178
1179         while(1) {
1180                 switch(c) {
1181                 case '\\': {
1182                         utf32 const tc = parse_escape_sequence();
1183                         if (tc >= 0x100) {
1184                                 warningf(&lexer_token.source_position,
1185                                                 "escape sequence out of range");
1186                         }
1187                         obstack_1grow(&symbol_obstack, tc);
1188                         break;
1189                 }
1190
1191                 case EOF: {
1192                         source_position_t source_position;
1193                         source_position.input_name = lexer_token.source_position.input_name;
1194                         source_position.linenr     = start_linenr;
1195                         errorf(&source_position, "string has no end");
1196                         lexer_token.type = T_ERROR;
1197                         return;
1198                 }
1199
1200                 case '"':
1201                         next_char();
1202                         goto end_of_string;
1203
1204                 default:
1205                         grow_symbol(c);
1206                         next_char();
1207                         break;
1208                 }
1209         }
1210
1211 end_of_string:
1212
1213         /* TODO: concatenate multiple strings separated by whitespace... */
1214
1215         /* add finishing 0 to the string */
1216         obstack_1grow(&symbol_obstack, '\0');
1217         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1218         const char *const string = obstack_finish(&symbol_obstack);
1219
1220 #if 0 /* TODO hash */
1221         /* check if there is already a copy of the string */
1222         result = strset_insert(&stringset, string);
1223         if(result != string) {
1224                 obstack_free(&symbol_obstack, string);
1225         }
1226 #else
1227         const char *const result = string;
1228 #endif
1229
1230         lexer_token.type           = T_STRING_LITERAL;
1231         lexer_token.v.string.begin = result;
1232         lexer_token.v.string.size  = size;
1233 }
1234
1235 /**
1236  * Parse a wide character constant and set lexer_token.
1237  */
1238 static void parse_wide_character_constant(void)
1239 {
1240         const unsigned start_linenr = lexer_token.source_position.linenr;
1241
1242         eat('\'');
1243
1244         while(1) {
1245                 switch(c) {
1246                 case '\\': {
1247                         wchar_rep_t tc = parse_escape_sequence();
1248                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1249                         break;
1250                 }
1251
1252                 MATCH_NEWLINE(
1253                         parse_error("newline while parsing character constant");
1254                         break;
1255                 )
1256
1257                 case '\'':
1258                         next_char();
1259                         goto end_of_wide_char_constant;
1260
1261                 case EOF: {
1262                         source_position_t source_position = lexer_token.source_position;
1263                         source_position.linenr = start_linenr;
1264                         errorf(&source_position, "EOF while parsing character constant");
1265                         lexer_token.type = T_ERROR;
1266                         return;
1267                 }
1268
1269                 default: {
1270                         wchar_rep_t tc = (wchar_rep_t) c;
1271                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1272                         next_char();
1273                         break;
1274                 }
1275                 }
1276         }
1277
1278 end_of_wide_char_constant:;
1279         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
1280         assert(size % sizeof(wchar_rep_t) == 0);
1281         size /= sizeof(wchar_rep_t);
1282
1283         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
1284
1285         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
1286         lexer_token.v.wide_string.begin = string;
1287         lexer_token.v.wide_string.size  = size;
1288         lexer_token.datatype            = type_wchar_t;
1289 }
1290
1291 /**
1292  * Parse a wide string literal and set lexer_token.
1293  */
1294 static void parse_wide_string_literal(void)
1295 {
1296         const unsigned start_linenr = lexer_token.source_position.linenr;
1297
1298         assert(c == '"');
1299         next_char();
1300
1301         while(1) {
1302                 switch(c) {
1303                 case '\\': {
1304                         wchar_rep_t tc = parse_escape_sequence();
1305                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1306                         break;
1307                 }
1308
1309                 case EOF: {
1310                         source_position_t source_position;
1311                         source_position.input_name = lexer_token.source_position.input_name;
1312                         source_position.linenr     = start_linenr;
1313                         errorf(&source_position, "string has no end");
1314                         lexer_token.type = T_ERROR;
1315                         return;
1316                 }
1317
1318                 case '"':
1319                         next_char();
1320                         goto end_of_string;
1321
1322                 default: {
1323                         wchar_rep_t tc = c;
1324                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1325                         next_char();
1326                         break;
1327                 }
1328                 }
1329         }
1330
1331 end_of_string:;
1332
1333         /* TODO: concatenate multiple strings separated by whitespace... */
1334
1335         /* add finishing 0 to the string */
1336         wchar_rep_t nul = L'\0';
1337         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1338         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1339         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1340
1341 #if 0 /* TODO hash */
1342         /* check if there is already a copy of the string */
1343         const wchar_rep_t *const result = strset_insert(&stringset, string);
1344         if(result != string) {
1345                 obstack_free(&symbol_obstack, string);
1346         }
1347 #else
1348         const wchar_rep_t *const result = string;
1349 #endif
1350
1351         lexer_token.type                = T_WIDE_STRING_LITERAL;
1352         lexer_token.v.wide_string.begin = result;
1353         lexer_token.v.wide_string.size  = size;
1354 }
1355
1356 /**
1357  * Parse a character constant and set lexer_token.
1358  */
1359 static void parse_character_constant(void)
1360 {
1361         const unsigned start_linenr = lexer_token.source_position.linenr;
1362
1363         eat('\'');
1364
1365         while(1) {
1366                 switch(c) {
1367                 case '\\': {
1368                         utf32 const tc = parse_escape_sequence();
1369                         if (tc >= 0x100) {
1370                                 warningf(&lexer_token.source_position,
1371                                                 "escape sequence out of range");
1372                         }
1373                         obstack_1grow(&symbol_obstack, tc);
1374                         break;
1375                 }
1376
1377                 MATCH_NEWLINE(
1378                         parse_error("newline while parsing character constant");
1379                         break;
1380                 )
1381
1382                 case '\'':
1383                         next_char();
1384                         goto end_of_char_constant;
1385
1386                 case EOF: {
1387                         source_position_t source_position;
1388                         source_position.input_name = lexer_token.source_position.input_name;
1389                         source_position.linenr     = start_linenr;
1390                         errorf(&source_position, "EOF while parsing character constant");
1391                         lexer_token.type = T_ERROR;
1392                         return;
1393                 }
1394
1395                 default:
1396                         grow_symbol(c);
1397                         next_char();
1398                         break;
1399
1400                 }
1401         }
1402
1403 end_of_char_constant:;
1404         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1405         const char *const string = obstack_finish(&symbol_obstack);
1406
1407         lexer_token.type           = T_CHARACTER_CONSTANT;
1408         lexer_token.v.string.begin = string;
1409         lexer_token.v.string.size  = size;
1410         lexer_token.datatype       = c_mode & _CXX && size == 1 ? type_char : type_int;
1411 }
1412
1413 /**
1414  * Skip a multiline comment.
1415  */
1416 static void skip_multiline_comment(void)
1417 {
1418         unsigned start_linenr = lexer_token.source_position.linenr;
1419
1420         while(1) {
1421                 switch(c) {
1422                 case '/':
1423                         next_char();
1424                         if (c == '*') {
1425                                 /* nested comment, warn here */
1426                                 if (warning.comment) {
1427                                         warningf(&lexer_token.source_position, "'/*' within comment");
1428                                 }
1429                         }
1430                         break;
1431                 case '*':
1432                         next_char();
1433                         if(c == '/') {
1434                                 next_char();
1435                                 return;
1436                         }
1437                         break;
1438
1439                 MATCH_NEWLINE(break;)
1440
1441                 case EOF: {
1442                         source_position_t source_position;
1443                         source_position.input_name = lexer_token.source_position.input_name;
1444                         source_position.linenr     = start_linenr;
1445                         errorf(&source_position, "at end of file while looking for comment end");
1446                         return;
1447                 }
1448
1449                 default:
1450                         next_char();
1451                         break;
1452                 }
1453         }
1454 }
1455
1456 /**
1457  * Skip a single line comment.
1458  */
1459 static void skip_line_comment(void)
1460 {
1461         while(1) {
1462                 switch(c) {
1463                 case EOF:
1464                         return;
1465
1466                 case '\n':
1467                 case '\r':
1468                         return;
1469
1470                 case '\\':
1471                         next_char();
1472                         if (c == '\n' || c == '\r') {
1473                                 if (warning.comment)
1474                                         warningf(&lexer_token.source_position, "multi-line comment");
1475                                 return;
1476                         }
1477                         break;
1478
1479                 default:
1480                         next_char();
1481                         break;
1482                 }
1483         }
1484 }
1485
1486 /** The current preprocessor token. */
1487 static token_t pp_token;
1488
1489 /**
1490  * Read the next preprocessor token.
1491  */
1492 static inline void next_pp_token(void)
1493 {
1494         lexer_next_preprocessing_token();
1495         pp_token = lexer_token;
1496 }
1497
1498 /**
1499  * Eat all preprocessor tokens until newline.
1500  */
1501 static void eat_until_newline(void)
1502 {
1503         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1504                 next_pp_token();
1505         }
1506 }
1507
1508 /**
1509  * Handle the define directive.
1510  */
1511 static void define_directive(void)
1512 {
1513         lexer_next_preprocessing_token();
1514         if(lexer_token.type != T_IDENTIFIER) {
1515                 parse_error("expected identifier after #define\n");
1516                 eat_until_newline();
1517         }
1518 }
1519
1520 /**
1521  * Handle the ifdef directive.
1522  */
1523 static void ifdef_directive(int is_ifndef)
1524 {
1525         (void) is_ifndef;
1526         lexer_next_preprocessing_token();
1527         //expect_identifier();
1528         //extect_newline();
1529 }
1530
1531 /**
1532  * Handle the endif directive.
1533  */
1534 static void endif_directive(void)
1535 {
1536         //expect_newline();
1537 }
1538
1539 /**
1540  * Parse the line directive.
1541  */
1542 static void parse_line_directive(void)
1543 {
1544         if(pp_token.type != T_INTEGER) {
1545                 parse_error("expected integer");
1546         } else {
1547                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1548                 next_pp_token();
1549         }
1550         if(pp_token.type == T_STRING_LITERAL) {
1551                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1552                 next_pp_token();
1553         }
1554
1555         eat_until_newline();
1556 }
1557
1558 /**
1559  * STDC pragmas.
1560  */
1561 typedef enum stdc_pragma_kind_t {
1562         STDC_UNKNOWN,
1563         STDC_FP_CONTRACT,
1564         STDC_FENV_ACCESS,
1565         STDC_CX_LIMITED_RANGE
1566 } stdc_pragma_kind_t;
1567
1568 /**
1569  * STDC pragma values.
1570  */
1571 typedef enum stdc_pragma_value_kind_t {
1572         STDC_VALUE_UNKNOWN,
1573         STDC_VALUE_ON,
1574         STDC_VALUE_OFF,
1575         STDC_VALUE_DEFAULT
1576 } stdc_pragma_value_kind_t;
1577
1578 /**
1579  * Parse a pragma directive.
1580  */
1581 static void parse_pragma(void) {
1582         bool unknown_pragma = true;
1583
1584         next_pp_token();
1585         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1586                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1587                 /* a STDC pragma */
1588                 if (c_mode & _C99) {
1589                         next_pp_token();
1590
1591                         switch (pp_token.v.symbol->pp_ID) {
1592                         case TP_FP_CONTRACT:
1593                                 kind = STDC_FP_CONTRACT;
1594                                 break;
1595                         case TP_FENV_ACCESS:
1596                                 kind = STDC_FENV_ACCESS;
1597                                 break;
1598                         case TP_CX_LIMITED_RANGE:
1599                                 kind = STDC_CX_LIMITED_RANGE;
1600                                 break;
1601                         default:
1602                                 break;
1603                         }
1604                         if (kind != STDC_UNKNOWN) {
1605                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1606                                 next_pp_token();
1607                                 switch (pp_token.v.symbol->pp_ID) {
1608                                 case TP_ON:
1609                                         value = STDC_VALUE_ON;
1610                                         break;
1611                                 case TP_OFF:
1612                                         value = STDC_VALUE_OFF;
1613                                         break;
1614                                 case TP_DEFAULT:
1615                                         value = STDC_VALUE_DEFAULT;
1616                                         break;
1617                                 default:
1618                                         break;
1619                                 }
1620                                 if (value != STDC_VALUE_UNKNOWN) {
1621                                         unknown_pragma = false;
1622                                 } else {
1623                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1624                                 }
1625                         }
1626                 }
1627         } else {
1628                 unknown_pragma = true;
1629         }
1630         eat_until_newline();
1631         if (unknown_pragma && warning.unknown_pragmas) {
1632                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1633         }
1634 }
1635
1636 /**
1637  * Parse a preprocessor non-null directive.
1638  */
1639 static void parse_preprocessor_identifier(void)
1640 {
1641         assert(pp_token.type == T_IDENTIFIER);
1642         symbol_t *symbol = pp_token.v.symbol;
1643
1644         switch(symbol->pp_ID) {
1645         case TP_include:
1646                 printf("include - enable header name parsing!\n");
1647                 break;
1648         case TP_define:
1649                 define_directive();
1650                 break;
1651         case TP_ifdef:
1652                 ifdef_directive(0);
1653                 break;
1654         case TP_ifndef:
1655                 ifdef_directive(1);
1656                 break;
1657         case TP_endif:
1658                 endif_directive();
1659                 break;
1660         case TP_line:
1661                 next_pp_token();
1662                 parse_line_directive();
1663                 break;
1664         case TP_if:
1665         case TP_else:
1666         case TP_elif:
1667         case TP_undef:
1668         case TP_error:
1669                 /* TODO; output the rest of the line */
1670                 parse_error("#error directive: ");
1671                 break;
1672         case TP_pragma:
1673                 parse_pragma();
1674                 break;
1675         }
1676 }
1677
1678 /**
1679  * Parse a preprocessor directive.
1680  */
1681 static void parse_preprocessor_directive(void)
1682 {
1683         next_pp_token();
1684
1685         switch(pp_token.type) {
1686         case T_IDENTIFIER:
1687                 parse_preprocessor_identifier();
1688                 break;
1689         case T_INTEGER:
1690                 parse_line_directive();
1691                 break;
1692         case '\n':
1693                 /* NULL directive, see § 6.10.7 */
1694                 break;
1695         default:
1696                 parse_error("invalid preprocessor directive");
1697                 eat_until_newline();
1698                 break;
1699         }
1700 }
1701
1702 #define MAYBE_PROLOG                                       \
1703                         next_char();                                   \
1704                         while(1) {                                     \
1705                                 switch(c) {
1706
1707 #define MAYBE(ch, set_type)                                \
1708                                 case ch:                                   \
1709                                         next_char();                           \
1710                                         lexer_token.type = set_type;           \
1711                                         return;
1712
1713 #define ELSE_CODE(code)                                    \
1714                                 default:                                   \
1715                                         code                                   \
1716                                 }                                          \
1717                         } /* end of while(1) */                        \
1718                         break;
1719
1720 #define ELSE(set_type)                                     \
1721                 ELSE_CODE(                                         \
1722                         lexer_token.type = set_type;                   \
1723                         return;                                        \
1724                 )
1725
1726 void lexer_next_preprocessing_token(void)
1727 {
1728         while(1) {
1729                 switch(c) {
1730                 case ' ':
1731                 case '\t':
1732                         next_char();
1733                         break;
1734
1735                 MATCH_NEWLINE(
1736                         lexer_token.type = '\n';
1737                         return;
1738                 )
1739
1740                 SYMBOL_CHARS
1741                         parse_symbol();
1742                         /* might be a wide string ( L"string" ) */
1743                         if(lexer_token.type == T_IDENTIFIER &&
1744                             lexer_token.v.symbol == symbol_L) {
1745                             if(c == '"') {
1746                                         parse_wide_string_literal();
1747                                 } else if(c == '\'') {
1748                                         parse_wide_character_constant();
1749                                 }
1750                         }
1751                         return;
1752
1753                 DIGITS
1754                         parse_number();
1755                         return;
1756
1757                 case '"':
1758                         parse_string_literal();
1759                         return;
1760
1761                 case '\'':
1762                         parse_character_constant();
1763                         return;
1764
1765                 case '.':
1766                         MAYBE_PROLOG
1767                                 DIGITS
1768                                         put_back(c);
1769                                         c = '.';
1770                                         parse_number_dec();
1771                                         return;
1772
1773                                 case '.':
1774                                         MAYBE_PROLOG
1775                                         MAYBE('.', T_DOTDOTDOT)
1776                                         ELSE_CODE(
1777                                                 put_back(c);
1778                                                 c = '.';
1779                                                 lexer_token.type = '.';
1780                                                 return;
1781                                         )
1782                         ELSE('.')
1783                 case '&':
1784                         MAYBE_PROLOG
1785                         MAYBE('&', T_ANDAND)
1786                         MAYBE('=', T_ANDEQUAL)
1787                         ELSE('&')
1788                 case '*':
1789                         MAYBE_PROLOG
1790                         MAYBE('=', T_ASTERISKEQUAL)
1791                         ELSE('*')
1792                 case '+':
1793                         MAYBE_PROLOG
1794                         MAYBE('+', T_PLUSPLUS)
1795                         MAYBE('=', T_PLUSEQUAL)
1796                         ELSE('+')
1797                 case '-':
1798                         MAYBE_PROLOG
1799                         MAYBE('>', T_MINUSGREATER)
1800                         MAYBE('-', T_MINUSMINUS)
1801                         MAYBE('=', T_MINUSEQUAL)
1802                         ELSE('-')
1803                 case '!':
1804                         MAYBE_PROLOG
1805                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1806                         ELSE('!')
1807                 case '/':
1808                         MAYBE_PROLOG
1809                         MAYBE('=', T_SLASHEQUAL)
1810                                 case '*':
1811                                         next_char();
1812                                         skip_multiline_comment();
1813                                         lexer_next_preprocessing_token();
1814                                         return;
1815                                 case '/':
1816                                         next_char();
1817                                         skip_line_comment();
1818                                         lexer_next_preprocessing_token();
1819                                         return;
1820                         ELSE('/')
1821                 case '%':
1822                         MAYBE_PROLOG
1823                         MAYBE('>', '}')
1824                         MAYBE('=', T_PERCENTEQUAL)
1825                                 case ':':
1826                                         MAYBE_PROLOG
1827                                                 case '%':
1828                                                         MAYBE_PROLOG
1829                                                         MAYBE(':', T_HASHHASH)
1830                                                         ELSE_CODE(
1831                                                                 put_back(c);
1832                                                                 c = '%';
1833                                                                 lexer_token.type = '#';
1834                                                                 return;
1835                                                         )
1836                                         ELSE('#')
1837                         ELSE('%')
1838                 case '<':
1839                         MAYBE_PROLOG
1840                         MAYBE(':', '[')
1841                         MAYBE('%', '{')
1842                         MAYBE('=', T_LESSEQUAL)
1843                                 case '<':
1844                                         MAYBE_PROLOG
1845                                         MAYBE('=', T_LESSLESSEQUAL)
1846                                         ELSE(T_LESSLESS)
1847                         ELSE('<')
1848                 case '>':
1849                         MAYBE_PROLOG
1850                         MAYBE('=', T_GREATEREQUAL)
1851                                 case '>':
1852                                         MAYBE_PROLOG
1853                                         MAYBE('=', T_GREATERGREATEREQUAL)
1854                                         ELSE(T_GREATERGREATER)
1855                         ELSE('>')
1856                 case '^':
1857                         MAYBE_PROLOG
1858                         MAYBE('=', T_CARETEQUAL)
1859                         ELSE('^')
1860                 case '|':
1861                         MAYBE_PROLOG
1862                         MAYBE('=', T_PIPEEQUAL)
1863                         MAYBE('|', T_PIPEPIPE)
1864                         ELSE('|')
1865                 case ':':
1866                         MAYBE_PROLOG
1867                         MAYBE('>', ']')
1868                         ELSE(':')
1869                 case '=':
1870                         MAYBE_PROLOG
1871                         MAYBE('=', T_EQUALEQUAL)
1872                         ELSE('=')
1873                 case '#':
1874                         MAYBE_PROLOG
1875                         MAYBE('#', T_HASHHASH)
1876                         ELSE('#')
1877
1878                 case '?':
1879                 case '[':
1880                 case ']':
1881                 case '(':
1882                 case ')':
1883                 case '{':
1884                 case '}':
1885                 case '~':
1886                 case ';':
1887                 case ',':
1888                 case '\\':
1889                         lexer_token.type = c;
1890                         next_char();
1891                         return;
1892
1893                 case EOF:
1894                         lexer_token.type = T_EOF;
1895                         return;
1896
1897                 default:
1898 dollar_sign:
1899                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1900                         next_char();
1901                         lexer_token.type = T_ERROR;
1902                         return;
1903                 }
1904         }
1905 }
1906
1907 void lexer_next_token(void)
1908 {
1909         lexer_next_preprocessing_token();
1910
1911         while (lexer_token.type == '\n') {
1912 newline_found:
1913                 lexer_next_preprocessing_token();
1914         }
1915
1916         if (lexer_token.type == '#') {
1917                 parse_preprocessor_directive();
1918                 goto newline_found;
1919         }
1920 }
1921
1922 void init_lexer(void)
1923 {
1924         strset_init(&stringset);
1925         symbol_L = symbol_table_insert("L");
1926 }
1927
1928 void lexer_open_stream(FILE *stream, const char *input_name)
1929 {
1930         input                                  = stream;
1931         lexer_token.source_position.linenr     = 0;
1932         lexer_token.source_position.input_name = input_name;
1933
1934         bufpos = NULL;
1935         bufend = NULL;
1936
1937         /* place a virtual \n at the beginning so the lexer knows that we're
1938          * at the beginning of a line */
1939         c = '\n';
1940 }
1941
1942 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1943 {
1944         input                                  = NULL;
1945         lexer_token.source_position.linenr     = 0;
1946         lexer_token.source_position.input_name = input_name;
1947
1948 #if 0 // TODO
1949         bufpos = buffer;
1950         bufend = buffer + len;
1951 #else
1952         (void)buffer;
1953         (void)len;
1954         panic("builtin lexing not done yet");
1955 #endif
1956
1957         /* place a virtual \n at the beginning so the lexer knows that we're
1958          * at the beginning of a line */
1959         c = '\n';
1960 }
1961
1962 void exit_lexer(void)
1963 {
1964         strset_destroy(&stringset);
1965 }
1966
1967 static __attribute__((unused))
1968 void dbg_pos(const source_position_t source_position)
1969 {
1970         fprintf(stdout, "%s:%u\n", source_position.input_name,
1971                 source_position.linenr);
1972         fflush(stdout);
1973 }