X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=7c610825bc24344e083cf5d71f97f1379ccc3278;hb=78cc65b14fc5c53a16368fe5975c70c83e969b19;hp=f23cb1177f767d07e56af098dfb5c8db92e98e87;hpb=b25e6207470812cdca4e99650505b1b5de9e3046;p=cparser

diff --git a/lexer.c b/lexer.c
index f23cb11..7c61082 100644
--- a/lexer.c
+++ b/lexer.c
@@ -1,54 +1,70 @@
 #include <config.h>
 
+#include "diagnostic.h"
 #include "lexer.h"
 #include "token_t.h"
 #include "symbol_table_t.h"
 #include "adt/error.h"
 #include "adt/strset.h"
+#include "adt/util.h"
+#include "types.h"
+#include "type_t.h"
+#include "target_architecture.h"
+#include "parser.h"
+#include "warning.h"
 
 #include <assert.h>
 #include <errno.h>
 #include <string.h>
+#include <stdbool.h>
 #include <ctype.h>
 
 //#define DEBUG_CHARS
 #define MAX_PUTBACK 3
 
+#ifdef _WIN32
+/* No strtold on windows and no replacement yet */
+#define strtold(s, e) strtod(s, e)
+#endif
+
+#if defined HAS_SIGNED_CHAR
+typedef signed char char_type;
+#elif defined HAS_UNSIGNED_CHAR
+typedef unsigned char char_type;
+#else
+#	error signedness of char not determined
+#endif
+
 static int         c;
 token_t            lexer_token;
+symbol_t          *symbol_L;
 static FILE       *input;
 static char        buf[1024 + MAX_PUTBACK];
 static const char *bufend;
 static const char *bufpos;
 static strset_t    stringset;
-//static FILE      **input_stack;
-//static char      **buf_stack;
 
-static
-void error_prefix_at(const char *input_name, unsigned linenr)
+static void error_prefix_at(const char *input_name, unsigned linenr)
 {
 	fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
 }
 
-static
-void error_prefix(void)
+static void error_prefix(void)
 {
 	error_prefix_at(lexer_token.source_position.input_name,
 	                lexer_token.source_position.linenr);
 }
 
-static
-void parse_error(const char *msg)
+static void parse_error(const char *msg)
 {
 	error_prefix();
 	fprintf(stderr, "%s\n", msg);
 }
 
-static inline
-void next_char(void)
+static inline void next_real_char(void)
 {
-	bufpos++;
-	if(bufpos >= bufend) {
+	assert(bufpos <= bufend);
+	if (bufpos >= bufend) {
 		size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
 		                 input);
 		if(s == 0) {
@@ -58,83 +74,95 @@ void next_char(void)
 		bufpos = buf + MAX_PUTBACK;
 		bufend = buf + MAX_PUTBACK + s;
 	}
-	c = *(bufpos);
-#ifdef DEBUG_CHARS
-	printf("nchar '%c'\n", c);
-#endif
+	c = *bufpos++;
 }
 
-static inline
-void put_back(int pc)
+static inline void put_back(int pc)
 {
-	char *p = (char*) bufpos - 1;
-	bufpos--;
-	assert(p >= buf);
-	*p = pc;
+	assert(bufpos > buf);
+	*(--bufpos - buf + buf) = (char) pc;
 
 #ifdef DEBUG_CHARS
 	printf("putback '%c'\n", pc);
 #endif
 }
 
+static inline void next_char(void);
 
-static
-int replace_trigraph(void)
+#define MATCH_NEWLINE(code)                   \
+	case '\r':                                \
+		next_char();                          \
+		if(c == '\n') {                       \
+			next_char();                      \
+		}                                     \
+		lexer_token.source_position.linenr++; \
+		code                                  \
+	case '\n':                                \
+		next_char();                          \
+		lexer_token.source_position.linenr++; \
+		code
+
+#define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
+
+static void maybe_concat_lines(void)
 {
-#define MATCH_TRIGRAPH(ch,replacement)           \
-	case ch:                                     \
-		c = replacement;                         \
-		return 1;
+	eat('\\');
 
 	switch(c) {
-	MATCH_TRIGRAPH('=', '#')
-	MATCH_TRIGRAPH('(', '[')
-	MATCH_TRIGRAPH('/', '\\')
-	MATCH_TRIGRAPH(')', ']')
-	MATCH_TRIGRAPH('\'', '^')
-	MATCH_TRIGRAPH('<', '{')
-	MATCH_TRIGRAPH('!', '|')
-	MATCH_TRIGRAPH('>', '}')
-	MATCH_TRIGRAPH('-', '~')
+	MATCH_NEWLINE(return;)
+
 	default:
 		break;
 	}
 
-	return 0;
+	put_back(c);
+	c = '\\';
 }
 
-#define SKIP_TRIGRAPHS(custom_putback, no_trigraph_code) \
-	case '?':                                  \
-		next_char();                           \
-		if(c != '?') {                         \
-			custom_putback;                    \
-			put_back(c);                       \
-			c = '?';                           \
-			no_trigraph_code;                  \
-		}                                      \
-		next_char();                           \
-		if(replace_trigraph()) {               \
-			break;                             \
-		}                                      \
-		custom_putback;                        \
-		put_back('?');                         \
-		put_back(c);                           \
-		c = '?';                               \
-		no_trigraph_code;
-
-#define EAT_NEWLINE(newline_code)              \
-	if(c == '\r') {                            \
-		next_char();                           \
-		if(c == '\n')                          \
-			next_char();                       \
-		lexer_token.source_position.linenr++;  \
-		newline_code;                          \
-	} else if(c == '\n') {                     \
-		next_char();                           \
-		lexer_token.source_position.linenr++;  \
-		newline_code;                          \
+static inline void next_char(void)
+{
+	next_real_char();
+
+	/* filter trigraphs */
+	if(UNLIKELY(c == '\\')) {
+		maybe_concat_lines();
+		goto end_of_next_char;
+	}
+
+	if(LIKELY(c != '?'))
+		goto end_of_next_char;
+
+	next_real_char();
+	if(LIKELY(c != '?')) {
+		put_back(c);
+		c = '?';
+		goto end_of_next_char;
+	}
+
+	next_real_char();
+	switch(c) {
+	case '=': c = '#'; break;
+	case '(': c = '['; break;
+	case '/': c = '\\'; maybe_concat_lines(); break;
+	case ')': c = ']'; break;
+	case '\'': c = '^'; break;
+	case '<': c = '{'; break;
+	case '!': c = '|'; break;
+	case '>': c = '}'; break;
+	case '-': c = '~'; break;
+	default:
+		put_back(c);
+		put_back('?');
+		c = '?';
+		break;
 	}
 
+end_of_next_char:;
+#ifdef DEBUG_CHARS
+	printf("nchar '%c'\n", c);
+#endif
+}
+
 #define SYMBOL_CHARS  \
 	case 'a':         \
 	case 'b':         \
@@ -202,47 +230,27 @@ int replace_trigraph(void)
 	case '8':         \
 	case '9':
 
-static
-void parse_symbol(void)
+static void parse_symbol(void)
 {
 	symbol_t *symbol;
 	char     *string;
 
-	obstack_1grow(&symbol_obstack, c);
+	obstack_1grow(&symbol_obstack, (char) c);
 	next_char();
 
 	while(1) {
 		switch(c) {
-		case '\\':
-			next_char();
-			EAT_NEWLINE(break;)
-			goto end_symbol;
-
 		DIGITS
 		SYMBOL_CHARS
-			obstack_1grow(&symbol_obstack, c);
+			obstack_1grow(&symbol_obstack, (char) c);
 			next_char();
 			break;
 
-		case '?':
-			next_char();
-			if(c != '?') {
-				put_back(c);
-				c = '?';
-				goto end_symbol;
-			}
-			next_char();
-			if(replace_trigraph())
-				break;
-			put_back('?');
-			put_back(c);
-			c = '?';
-			goto end_symbol;
-
 		default:
 			goto end_symbol;
 		}
 	}
+
 end_symbol:
 	obstack_1grow(&symbol_obstack, '\0');
 
@@ -257,128 +265,352 @@ end_symbol:
 	}
 }
 
-static
-void parse_number_hex(void)
+static void parse_integer_suffix(bool is_oct_hex)
 {
-	assert(c == 'x' || c == 'X');
-	next_char();
+	bool is_unsigned  = false;
+	bool min_long     = false;
+	bool min_longlong = false;
 
-	if (!isdigit(c) &&
-		!('A' <= c && c <= 'F') &&
-		!('a' <= c && c <= 'f')) {
-		parse_error("premature end of hex number literal");
-		lexer_token.type = T_ERROR;
-		return;
+	if(c == 'U' || c == 'u') {
+		is_unsigned = true;
+		next_char();
+		if(c == 'L' || c == 'l') {
+			min_long = true;
+			next_char();
+			if(c == 'L' || c == 'l') {
+				min_longlong = true;
+				next_char();
+			}
+		}
+	} else if(c == 'l' || c == 'L') {
+		min_long = true;
+		next_char();
+		if(c == 'l' || c == 'L') {
+			min_longlong = true;
+			next_char();
+			if(c == 'u' || c == 'U') {
+				is_unsigned = true;
+				next_char();
+			}
+		} else if(c == 'u' || c == 'U') {
+			is_unsigned = true;
+			next_char();
+			lexer_token.datatype = type_unsigned_long;
+		}
 	}
 
-	int value = 0;
-	while(1) {
-		if (isdigit(c)) {
-			value = 16 * value + c - '0';
-		} else if ('A' <= c && c <= 'F') {
-			value = 16 * value + c - 'A' + 10;
-		} else if ('a' <= c && c <= 'f') {
-			value = 16 * value + c - 'a' + 10;
-		} else {
-			lexer_token.type     = T_INTEGER;
-			lexer_token.v.intvalue = value;
+	if(!is_unsigned) {
+		long long v = lexer_token.v.intvalue;
+		if(!min_long) {
+			if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
+				lexer_token.datatype = type_int;
+				return;
+			} else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
+				lexer_token.datatype = type_unsigned_int;
+				return;
+			}
+		}
+		if(!min_longlong) {
+			if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
+				lexer_token.datatype = type_long;
+				return;
+			} else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
+				lexer_token.datatype = type_unsigned_long;
+				return;
+			}
+		}
+		unsigned long long uv = (unsigned long long) v;
+		if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
+			lexer_token.datatype = type_unsigned_long_long;
 			return;
 		}
-		next_char();
+
+		lexer_token.datatype = type_long_long;
+	} else {
+		unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
+		if(!min_long && v <= TARGET_UINT_MAX) {
+			lexer_token.datatype = type_unsigned_int;
+			return;
+		}
+		if(!min_longlong && v <= TARGET_ULONG_MAX) {
+			lexer_token.datatype = type_unsigned_long;
+			return;
+		}
+		lexer_token.datatype = type_unsigned_long_long;
 	}
 }
 
-static
-void parse_number_oct(void)
+static void parse_floating_suffix(void)
 {
-	assert(c == 'o' || c == 'O');
-	next_char();
+	switch(c) {
+	/* TODO: do something usefull with the suffixes... */
+	case 'f':
+	case 'F':
+		next_char();
+		lexer_token.datatype = type_float;
+		break;
+	case 'l':
+	case 'L':
+		next_char();
+		lexer_token.datatype = type_long_double;
+		break;
+	default:
+		lexer_token.datatype = type_double;
+		break;
+	}
+}
 
-	int value = 0;
-	while(1) {
-		if ('0' <= c && c <= '7') {
-			value = 8 * value + c - '0';
-		} else {
-			lexer_token.type       = T_INTEGER;
-			lexer_token.v.intvalue = value;
-			return;
+/**
+ * A replacement for strtoull. Only those parts needed for
+ * our parser are implemented.
+ */
+static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
+	unsigned long long v = 0;
+
+	switch (base) {
+	case 16:
+		for (;; ++s) {
+			/* check for overrun */
+			if (v >= 0x1000000000000000ULL)
+				break;
+			switch (tolower(*s)) {
+			case '0': v <<= 4; break;
+			case '1': v <<= 4; v |= 0x1; break;
+			case '2': v <<= 4; v |= 0x2; break;
+			case '3': v <<= 4; v |= 0x3; break;
+			case '4': v <<= 4; v |= 0x4; break;
+			case '5': v <<= 4; v |= 0x5; break;
+			case '6': v <<= 4; v |= 0x6; break;
+			case '7': v <<= 4; v |= 0x7; break;
+			case '8': v <<= 4; v |= 0x8; break;
+			case '9': v <<= 4; v |= 0x9; break;
+			case 'a': v <<= 4; v |= 0xa; break;
+			case 'b': v <<= 4; v |= 0xb; break;
+			case 'c': v <<= 4; v |= 0xc; break;
+			case 'd': v <<= 4; v |= 0xd; break;
+			case 'e': v <<= 4; v |= 0xe; break;
+			case 'f': v <<= 4; v |= 0xf; break;
+			default:
+				goto end;
+			}
 		}
-		next_char();
+		break;
+	case 8:
+		for (;; ++s) {
+			/* check for overrun */
+			if (v >= 0x2000000000000000ULL)
+				break;
+			switch (tolower(*s)) {
+			case '0': v <<= 3; break;
+			case '1': v <<= 3; v |= 1; break;
+			case '2': v <<= 3; v |= 2; break;
+			case '3': v <<= 3; v |= 3; break;
+			case '4': v <<= 3; v |= 4; break;
+			case '5': v <<= 3; v |= 5; break;
+			case '6': v <<= 3; v |= 6; break;
+			case '7': v <<= 3; v |= 7; break;
+			default:
+				goto end;
+			}
+		}
+		break;
+	case 10:
+		for (;; ++s) {
+			/* check for overrun */
+			if (v > 0x1999999999999999ULL)
+				break;
+			switch (tolower(*s)) {
+			case '0': v *= 10; break;
+			case '1': v *= 10; v += 1; break;
+			case '2': v *= 10; v += 2; break;
+			case '3': v *= 10; v += 3; break;
+			case '4': v *= 10; v += 4; break;
+			case '5': v *= 10; v += 5; break;
+			case '6': v *= 10; v += 6; break;
+			case '7': v *= 10; v += 7; break;
+			case '8': v *= 10; v += 8; break;
+			case '9': v *= 10; v += 9; break;
+			default:
+				goto end;
+			}
+		}
+		break;
+	default:
+		assert(0);
+		break;
 	}
+end:
+	*endptr = s;
+	return v;
 }
 
-static
-void parse_number_dec(int first_char)
+static void parse_number_hex(void)
 {
-	int value = 0;
-	if(first_char > 0) {
-		assert(first_char >= '0' && first_char <= '9');
-		value = first_char - '0';
+	assert(c == 'x' || c == 'X');
+	next_char();
+
+	while(isxdigit(c)) {
+		obstack_1grow(&symbol_obstack, (char) c);
+		next_char();
 	}
+	obstack_1grow(&symbol_obstack, '\0');
+	char *string = obstack_finish(&symbol_obstack);
 
-	for(;;) {
-		if (isdigit(c)) {
-			value = 10 * value + c - '0';
-		} else {
-			lexer_token.type       = T_INTEGER;
-			lexer_token.v.intvalue = value;
-			return;
-		}
+	if(c == '.' || c == 'p' || c == 'P') {
 		next_char();
+		panic("Hex floating point numbers not implemented yet");
 	}
+	if(*string == '\0') {
+		parse_error("invalid hex number");
+		lexer_token.type = T_ERROR;
+	}
+
+	const char *endptr;
+	lexer_token.type       = T_INTEGER;
+	lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
+	if(*endptr != '\0') {
+		parse_error("hex number literal too long");
+	}
+
+	obstack_free(&symbol_obstack, string);
+	parse_integer_suffix(true);
 }
 
-static
-void parse_number(void)
+static inline bool is_octal_digit(int chr)
 {
-	// TODO check for overflow
-	// TODO check for various invalid inputs sequences
+	return '0' <= chr && chr <= '7';
+}
 
-	if (c == '0') {
+static void parse_number_oct(void)
+{
+	while(is_octal_digit(c)) {
+		obstack_1grow(&symbol_obstack, (char) c);
 		next_char();
-		switch (c) {
-			case 'X':
-			case 'x': parse_number_hex(); break;
-			case 'o':
-			case 'O': parse_number_oct(); break;
-			default:  parse_number_dec('0');
-		}
-	} else {
-		parse_number_dec(0);
 	}
-	if(c == 'U' || c == 'U') {
-		/* TODO do something with the suffixes... */
+	obstack_1grow(&symbol_obstack, '\0');
+	char *string = obstack_finish(&symbol_obstack);
+
+	const char *endptr;
+	lexer_token.type       = T_INTEGER;
+	lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
+	if(*endptr != '\0') {
+		parse_error("octal number literal too long");
+	}
+
+	obstack_free(&symbol_obstack, string);
+	parse_integer_suffix(true);
+}
+
+static void parse_number_dec(void)
+{
+	bool is_float = false;
+	while(isdigit(c)) {
+		obstack_1grow(&symbol_obstack, (char) c);
 		next_char();
-		if(c == 'L' || c == 'l') {
+	}
+
+	if(c == '.') {
+		obstack_1grow(&symbol_obstack, '.');
+		next_char();
+
+		while(isdigit(c)) {
+			obstack_1grow(&symbol_obstack, (char) c);
 			next_char();
-			if(c == 'L' || c == 'l') {
-				next_char();
-			}
 		}
-	} else if(c == 'l' || c == 'L') {
+		is_float = true;
+	}
+	if(c == 'e' || c == 'E') {
+		obstack_1grow(&symbol_obstack, 'e');
 		next_char();
-		if(c == 'l' || c == 'L') {
+
+		if(c == '-' || c == '+') {
+			obstack_1grow(&symbol_obstack, (char) c);
 			next_char();
-			if(c == 'u' || c == 'U') {
-				next_char();
-			}
-		} else if(c == 'u' || c == 'U') {
+		}
+
+		while(isdigit(c)) {
+			obstack_1grow(&symbol_obstack, (char) c);
 			next_char();
 		}
+		is_float = true;
+	}
+
+	obstack_1grow(&symbol_obstack, '\0');
+	char *string = obstack_finish(&symbol_obstack);
+
+	if(is_float) {
+		char *endptr;
+		lexer_token.type         = T_FLOATINGPOINT;
+		lexer_token.v.floatvalue = strtold(string, &endptr);
+
+		if(*endptr != '\0') {
+			parse_error("invalid number literal");
+		}
+
+		parse_floating_suffix();
+	} else {
+		const char *endptr;
+		lexer_token.type       = T_INTEGER;
+		lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
+
+		if(*endptr != '\0') {
+			parse_error("invalid number literal");
+		}
+
+		parse_integer_suffix(false);
 	}
+	obstack_free(&symbol_obstack, string);
 }
 
-static int parse_octal_sequence(void)
+static void parse_number(void)
 {
-	int value = 0;
-	while(1) {
-		if(c < '0' || c > '7')
-			break;
-		value = 8 * value + c - '0';
+	if (c == '0') {
 		next_char();
+		switch (c) {
+			case 'X':
+			case 'x':
+				parse_number_hex();
+				break;
+			case '0':
+			case '1':
+			case '2':
+			case '3':
+			case '4':
+			case '5':
+			case '6':
+			case '7':
+				parse_number_oct();
+				break;
+			case '8':
+			case '9':
+				next_char();
+				parse_error("invalid octal number");
+				lexer_token.type = T_ERROR;
+				return;
+			case '.':
+			case 'e':
+			case 'E':
+			default:
+				obstack_1grow(&symbol_obstack, '0');
+				parse_number_dec();
+				return;
+		}
+	} else {
+		parse_number_dec();
 	}
+}
 
-	return value;
+static int parse_octal_sequence(const int first_digit)
+{
+	assert(is_octal_digit(first_digit));
+	int value = first_digit - '0';
+	if (!is_octal_digit(c)) return value;
+	value = 8 * value + c - '0';
+	next_char();
+	if (!is_octal_digit(c)) return value;
+	value = 8 * value + c - '0';
+	next_char();
+	return (char_type)value;
 }
 
 static int parse_hex_sequence(void)
@@ -397,102 +629,82 @@ static int parse_hex_sequence(void)
 		next_char();
 	}
 
-	return value;
+	return (char_type)value;
 }
 
 static int parse_escape_sequence(void)
 {
-	while(1) {
-		int ec = c;
-		next_char();
+	eat('\\');
 
-		switch(ec) {
-		case '"': return '"';
-		case '\'': return'\'';
-		case '\\':
-			EAT_NEWLINE(break;)
-			return '\\';
-		case 'a': return '\a';
-		case 'b': return '\b';
-		case 'f': return '\f';
-		case 'n': return '\n';
-		case 'r': return '\r';
-		case 't': return '\t';
-		case 'v': return '\v';
-		case 'x':
-			return parse_hex_sequence();
-		case '0':
-		case '1':
-		case '2':
-		case '3':
-		case '4':
-		case '5':
-		case '6':
-		case '7':
-			return parse_octal_sequence();
-		case '?':
-			if(c != '?') {
-				return '?';
-			}
-			/* might be a trigraph */
-			next_char();
-			if(replace_trigraph()) {
-				break;
-			}
-			put_back(c);
-			c = '?';
-			return '?';
+	int ec = c;
+	next_char();
 
-		case EOF:
-			parse_error("reached end of file while parsing escape sequence");
-			return EOF;
-		default:
-			parse_error("unknown escape sequence");
-			return EOF;
-		}
+	switch(ec) {
+	case '"':  return '"';
+	case '\'': return '\'';
+	case '\\': return '\\';
+	case '?': return '\?';
+	case 'a': return '\a';
+	case 'b': return '\b';
+	case 'f': return '\f';
+	case 'n': return '\n';
+	case 'r': return '\r';
+	case 't': return '\t';
+	case 'v': return '\v';
+	case 'x':
+		return parse_hex_sequence();
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+		return parse_octal_sequence(ec);
+	case EOF:
+		parse_error("reached end of file while parsing escape sequence");
+		return EOF;
+	default:
+		parse_error("unknown escape sequence");
+		return EOF;
 	}
 }
 
-const char *concat_strings(const char *s1, const char *s2)
+string_t concat_strings(const string_t *const s1, const string_t *const s2)
 {
-	size_t  len1   = strlen(s1);
-	size_t  len2   = strlen(s2);
+	const size_t len1 = s1->size - 1;
+	const size_t len2 = s2->size - 1;
 
-	char   *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
-	memcpy(concat, s1, len1);
-	memcpy(concat + len1, s2, len2 + 1);
+	char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
+	memcpy(concat, s1->begin, len1);
+	memcpy(concat + len1, s2->begin, len2 + 1);
 
+#if 0 /* TODO hash */
 	const char *result = strset_insert(&stringset, concat);
 	if(result != concat) {
 		obstack_free(&symbol_obstack, concat);
 	}
 
 	return result;
+#else
+	return (string_t){ concat, len1 + len2 + 1 };
+#endif
 }
 
-static
-void parse_string_literal(void)
+static void parse_string_literal(void)
 {
-	unsigned    start_linenr = lexer_token.source_position.linenr;
-	char       *string;
-	const char *result;
+	const unsigned start_linenr = lexer_token.source_position.linenr;
 
 	assert(c == '"');
 	next_char();
 
+	int tc;
 	while(1) {
 		switch(c) {
-		SKIP_TRIGRAPHS(,
-			obstack_1grow(&symbol_obstack, '?');
-			next_char();
-			break;
-		)
-
 		case '\\':
-			next_char();
-			EAT_NEWLINE(break;)
-			int ec = parse_escape_sequence();
-			obstack_1grow(&symbol_obstack, ec);
+			tc = parse_escape_sequence();
+			obstack_1grow(&symbol_obstack, (char) tc);
 			break;
 
 		case EOF:
@@ -507,7 +719,7 @@ void parse_string_literal(void)
 			goto end_of_string;
 
 		default:
-			obstack_1grow(&symbol_obstack, c);
+			obstack_1grow(&symbol_obstack, (char) c);
 			next_char();
 			break;
 		}
@@ -519,49 +731,136 @@ end_of_string:
 
 	/* add finishing 0 to the string */
 	obstack_1grow(&symbol_obstack, '\0');
-	string = obstack_finish(&symbol_obstack);
+	const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
+	const char *const string = obstack_finish(&symbol_obstack);
 
+#if 0 /* TODO hash */
 	/* check if there is already a copy of the string */
 	result = strset_insert(&stringset, string);
 	if(result != string) {
 		obstack_free(&symbol_obstack, string);
 	}
+#else
+	const char *const result = string;
+#endif
 
-	lexer_token.type     = T_STRING_LITERAL;
-	lexer_token.v.string = result;
+	lexer_token.type           = T_STRING_LITERAL;
+	lexer_token.v.string.begin = result;
+	lexer_token.v.string.size  = size;
 }
 
-#define MATCH_NEWLINE(code)                   \
-	case '\r':                                \
-		next_char();                          \
-		if(c == '\n') {                       \
-			next_char();                      \
-		}                                     \
-		lexer_token.source_position.linenr++; \
-		code;                                 \
-	case '\n':                                \
-		next_char();                          \
-		lexer_token.source_position.linenr++; \
-		code;
-
-static
-void parse_character_constant(void)
+static void parse_wide_character_constant(void)
 {
-	assert(c == '\'');
-	next_char();
+	eat('\'');
 
 	int found_char = 0;
 	while(1) {
 		switch(c) {
-		SKIP_TRIGRAPHS(,
-			next_char();
-			found_char = '?';
+		case '\\':
+			found_char = parse_escape_sequence();
+			break;
+
+		MATCH_NEWLINE(
+			parse_error("newline while parsing character constant");
 			break;
 		)
 
-		case '\\':
+		case '\'':
 			next_char();
-			EAT_NEWLINE(break;)
+			goto end_of_wide_char_constant;
+
+		case EOF:
+			parse_error("EOF while parsing character constant");
+			lexer_token.type = T_ERROR;
+			return;
+
+		default:
+			if(found_char != 0) {
+				parse_error("more than 1 characters in character "
+				            "constant");
+				goto end_of_wide_char_constant;
+			} else {
+				found_char = c;
+				next_char();
+			}
+			break;
+		}
+	}
+
+end_of_wide_char_constant:
+	lexer_token.type       = T_INTEGER;
+	lexer_token.v.intvalue = found_char;
+	lexer_token.datatype   = type_wchar_t;
+}
+
+static void parse_wide_string_literal(void)
+{
+	const unsigned start_linenr = lexer_token.source_position.linenr;
+
+	assert(c == '"');
+	next_char();
+
+	while(1) {
+		switch(c) {
+			case '\\': {
+				wchar_rep_t tc = parse_escape_sequence();
+				obstack_grow(&symbol_obstack, &tc, sizeof(tc));
+				break;
+			}
+
+			case EOF:
+				error_prefix_at(lexer_token.source_position.input_name,
+				                start_linenr);
+				fprintf(stderr, "string has no end\n");
+				lexer_token.type = T_ERROR;
+				return;
+
+			case '"':
+				next_char();
+				goto end_of_string;
+
+			default: {
+				wchar_rep_t tc = c;
+				obstack_grow(&symbol_obstack, &tc, sizeof(tc));
+				next_char();
+				break;
+			}
+		}
+	}
+
+end_of_string:;
+
+	/* TODO: concatenate multiple strings separated by whitespace... */
+
+	/* add finishing 0 to the string */
+	wchar_rep_t nul = L'\0';
+	obstack_grow(&symbol_obstack, &nul, sizeof(nul));
+	const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
+	const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
+
+#if 0 /* TODO hash */
+	/* check if there is already a copy of the string */
+	const wchar_rep_t *const result = strset_insert(&stringset, string);
+	if(result != string) {
+		obstack_free(&symbol_obstack, string);
+	}
+#else
+	const wchar_rep_t *const result = string;
+#endif
+
+	lexer_token.type                = T_WIDE_STRING_LITERAL;
+	lexer_token.v.wide_string.begin = result;
+	lexer_token.v.wide_string.size  = size;
+}
+
+static void parse_character_constant(void)
+{
+	eat('\'');
+
+	int found_char = 0;
+	while(1) {
+		switch(c) {
+		case '\\':
 			found_char = parse_escape_sequence();
 			break;
 
@@ -595,94 +894,47 @@ void parse_character_constant(void)
 end_of_char_constant:
 	lexer_token.type       = T_INTEGER;
 	lexer_token.v.intvalue = found_char;
+	lexer_token.datatype   = type_int;
 }
 
-static
-void skip_multiline_comment(void)
+static void skip_multiline_comment(void)
 {
 	unsigned start_linenr = lexer_token.source_position.linenr;
-	int had_star = 0;
 
 	while(1) {
 		switch(c) {
 		case '*':
 			next_char();
-			had_star = 1;
-			break;
-
-		case '/':
-			next_char();
-			if(had_star) {
+			if(c == '/') {
+				next_char();
 				return;
 			}
-			had_star = 0;
-			break;
-
-		case '\\':
-			next_char();
-			EAT_NEWLINE(break;)
-			had_star = 0;
-			break;
-
-		case '?':
-			next_char();
-			if(c != '?') {
-				had_star = 0;
-				break;
-			}
-			next_char();
-			if(replace_trigraph())
-				break;
-			put_back(c);
-			c = '?';
-			had_star = 0;
-			/* we don't put back the 2nd ? as the comment text is discarded
-			 * anyway */
 			break;
 
-		MATCH_NEWLINE(had_star = 0; break;)
+		MATCH_NEWLINE(break;)
 
 		case EOF:
 			error_prefix_at(lexer_token.source_position.input_name,
 			                start_linenr);
 			fprintf(stderr, "at end of file while looking for comment end\n");
 			return;
+
 		default:
-			had_star = 0;
 			next_char();
 			break;
 		}
 	}
 }
 
-static
-void skip_line_comment(void)
+static void skip_line_comment(void)
 {
 	while(1) {
 		switch(c) {
-		case '?':
-			next_char();
-			if(c != '?')
-				break;
-			next_char();
-			if(replace_trigraph())
-				break;
-			put_back('?');
-			/* we don't put back the 2nd ? as the comment text is discarded
-			 * anyway */
-			break;
-
-		case '\\':
-			next_char();
-			if(c == '\n') {
-				next_char();
-				lexer_token.source_position.linenr++;
-			}
-			break;
-
 		case EOF:
-		case '\r':
+			return;
+
 		case '\n':
+		case '\r':
 			return;
 
 		default:
@@ -694,23 +946,20 @@ void skip_line_comment(void)
 
 static token_t pp_token;
 
-static inline
-void next_pp_token(void)
+static inline void next_pp_token(void)
 {
 	lexer_next_preprocessing_token();
 	pp_token = lexer_token;
 }
 
-static
-void eat_until_newline(void)
+static void eat_until_newline(void)
 {
 	while(pp_token.type != '\n' && pp_token.type != T_EOF) {
 		next_pp_token();
 	}
 }
 
-static
-void error_directive(void)
+static void error_directive(void)
 {
 	error_prefix();
 	fprintf(stderr, "#error directive: \n");
@@ -718,8 +967,7 @@ void error_directive(void)
 	/* parse pp-tokens until new-line */
 }
 
-static
-void define_directive(void)
+static void define_directive(void)
 {
 	lexer_next_preprocessing_token();
 	if(lexer_token.type != T_IDENTIFIER) {
@@ -728,8 +976,7 @@ void define_directive(void)
 	}
 }
 
-static
-void ifdef_directive(int is_ifndef)
+static void ifdef_directive(int is_ifndef)
 {
 	(void) is_ifndef;
 	lexer_next_preprocessing_token();
@@ -737,31 +984,28 @@ void ifdef_directive(int is_ifndef)
 	//extect_newline();
 }
 
-static
-void endif_directive(void)
+static void endif_directive(void)
 {
 	//expect_newline();
 }
 
-static
-void parse_line_directive(void)
+static void parse_line_directive(void)
 {
 	if(pp_token.type != T_INTEGER) {
 		parse_error("expected integer");
 	} else {
-		lexer_token.source_position.linenr = pp_token.v.intvalue - 1;
+		lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
 		next_pp_token();
 	}
 	if(pp_token.type == T_STRING_LITERAL) {
-		lexer_token.source_position.input_name = pp_token.v.string;
+		lexer_token.source_position.input_name = pp_token.v.string.begin;
 		next_pp_token();
 	}
 
 	eat_until_newline();
 }
 
-static
-void parse_preprocessor_identifier(void)
+static void parse_preprocessor_identifier(void)
 {
 	assert(pp_token.type == T_IDENTIFIER);
 	symbol_t *symbol = pp_token.v.symbol;
@@ -794,12 +1038,15 @@ void parse_preprocessor_identifier(void)
 		error_directive();
 		break;
 	case TP_pragma:
+		if (warning.unknown_pragmas) {
+			warningf(lexer_token.source_position, "encountered unknown #pragma");
+		}
+		eat_until_newline();
 		break;
 	}
 }
 
-static
-void parse_preprocessor_directive()
+static void parse_preprocessor_directive(void)
 {
 	next_pp_token();
 
@@ -829,14 +1076,6 @@ void parse_preprocessor_directive()
 					return;
 
 #define ELSE_CODE(code)                                    \
-				SKIP_TRIGRAPHS(,                           \
-					code;                                  \
-				)                                          \
-														   \
-				case '\\':                                 \
-					next_char();                           \
-					EAT_NEWLINE(break;)                    \
-					/* fallthrough */                      \
 				default:                                   \
 					code;                                  \
 				}                                          \
@@ -865,6 +1104,15 @@ void lexer_next_preprocessing_token(void)
 
 		SYMBOL_CHARS
 			parse_symbol();
+			/* might be a wide string ( L"string" ) */
+			if(lexer_token.type == T_IDENTIFIER &&
+			    lexer_token.v.symbol == symbol_L) {
+			    if(c == '"') {
+					parse_wide_string_literal();
+				} else if(c == '\'') {
+					parse_wide_character_constant();
+				}
+			}
 			return;
 
 		DIGITS
@@ -879,20 +1127,23 @@ void lexer_next_preprocessing_token(void)
 			parse_character_constant();
 			return;
 
-		case '\\':
-			next_char();
-			if(c == '\n') {
-				next_char();
-				lexer_token.source_position.linenr++;
-				break;
-			} else {
-				parse_error("unexpected '\\' found");
-				lexer_token.type = T_ERROR;
-			}
-			return;
-
 		case '.':
 			MAYBE_PROLOG
+				case '0':
+				case '1':
+				case '2':
+				case '3':
+				case '4':
+				case '5':
+				case '6':
+				case '7':
+				case '8':
+				case '9':
+					put_back(c);
+					c = '.';
+					parse_number_dec();
+					return;
+
 				case '.':
 					MAYBE_PROLOG
 					MAYBE('.', T_DOTDOTDOT)
@@ -999,22 +1250,6 @@ void lexer_next_preprocessing_token(void)
 			ELSE('#')
 
 		case '?':
-			next_char();
-			/* just a simple ? */
-			if(c != '?') {
-				lexer_token.type = '?';
-				return;
-			}
-			/* might be a trigraph */
-			next_char();
-			if(replace_trigraph()) {
-				break;
-			}
-			put_back(c);
-			c = '?';
-			lexer_token.type = '?';
-			return;
-
 		case '[':
 		case ']':
 		case '(':
@@ -1024,6 +1259,7 @@ void lexer_next_preprocessing_token(void)
 		case '~':
 		case ';':
 		case ',':
+		case '\\':
 			lexer_token.type = c;
 			next_char();
 			return;
@@ -1070,8 +1306,12 @@ void lexer_open_stream(FILE *stream, const char *input_name)
 	lexer_token.source_position.linenr     = 0;
 	lexer_token.source_position.input_name = input_name;
 
-	/* we place a virtual '\n' at the beginning so the lexer knows we're at the
-	 * beginning of a line */
+	symbol_L = symbol_table_insert("L");
+	bufpos = NULL;
+	bufend = NULL;
+
+	/* place a virtual \n at the beginning so the lexer knows that we're
+	 * at the beginning of a line */
 	c = '\n';
 }
 
@@ -1083,7 +1323,7 @@ void exit_lexer(void)
 static __attribute__((unused))
 void dbg_pos(const source_position_t source_position)
 {
-	fprintf(stdout, "%s:%d\n", source_position.input_name,
+	fprintf(stdout, "%s:%u\n", source_position.input_name,
 	        source_position.linenr);
 	fflush(stdout);
 }