2 * This file is part of cparser.
3 * Copyright (C) 2012 Matthias Braun <matze@braunis.de>
7 #include "diagnostic.h"
12 typedef size_t (*decode_func)(input_t *input, utf32 *buffer, size_t buffer_size);
27 /* state for utf-8 decoder */
28 utf32 utf8_part_decoded_min_code;
29 utf32 utf8_part_decoded_char;
30 size_t utf8_part_decoded_rest_len;
33 static input_error_callback_func input_error;
35 void set_input_error_callback(input_error_callback_func new_func)
37 input_error = new_func;
40 static size_t read_block(input_t *input, unsigned char *const read_buf,
43 if (input->kind == INPUT_FILE) {
44 FILE *file = input->in.file;
45 size_t const s = fread(read_buf, 1, n, file);
47 /* on OS/X ferror appears to return true on eof as well when running
48 * the application in gdb... */
49 if (!feof(file) && ferror(file))
50 input_error(0, 0, "read from input failed");
55 assert(input->kind == INPUT_STRING);
56 size_t len = strlen(input->in.string);
59 memcpy(read_buf, input->in.string, len);
60 input->in.string += len;
65 static size_t decode_iso_8859_1(input_t *input, utf32 *buffer,
68 unsigned char read_buf[buffer_size];
69 size_t const s = read_block(input, read_buf, sizeof(read_buf));
71 unsigned char const *src = read_buf;
72 unsigned char const *end = read_buf + s;
80 static size_t decode_iso_8859_15(input_t *input, utf32 *buffer,
83 unsigned char read_buf[buffer_size];
84 size_t const s = read_block(input, read_buf, sizeof(read_buf));
86 unsigned char const *src = read_buf;
87 unsigned char const *end = read_buf + s;
92 case 0xA4: tc = 0x20AC; break; // €
93 case 0xA6: tc = 0x0160; break; // Š
94 case 0xA8: tc = 0x0161; break; // š
95 case 0xB4: tc = 0x017D; break; // Ž
96 case 0xB8: tc = 0x017E; break; // ž
97 case 0xBC: tc = 0x0152; break; // Œ
98 case 0xBD: tc = 0x0153; break; // œ
99 case 0xBE: tc = 0x0178; break; // Ÿ
107 static size_t decode_utf8(input_t *input, utf32 *buffer, size_t buffer_size)
109 unsigned char read_buf[buffer_size];
112 size_t const s = read_block(input, read_buf, sizeof(read_buf));
114 if (input->utf8_part_decoded_rest_len > 0)
115 input_error(0, 0, "incomplete input char at end of input");
119 unsigned char const *src = read_buf;
120 unsigned char const *end = read_buf + s;
125 if (input->utf8_part_decoded_rest_len != 0) {
126 min_code = input->utf8_part_decoded_min_code;
127 decoded = input->utf8_part_decoded_char;
128 size_t const rest_len = input->utf8_part_decoded_rest_len;
129 input->utf8_part_decoded_rest_len = 0;
131 case 4: goto realign;
132 case 3: goto three_more;
133 case 2: goto two_more;
134 default: goto one_more;
139 if ((*src & 0x80) == 0) {
141 } else if ((*src & 0xE0) == 0xC0) {
143 decoded = *src++ & 0x1F;
146 input->utf8_part_decoded_min_code = min_code;
147 input->utf8_part_decoded_char = decoded;
148 input->utf8_part_decoded_rest_len = 1;
151 if ((*src & 0xC0) == 0x80) {
152 decoded = (decoded << 6) | (*src++ & 0x3F);
156 if (decoded < min_code ||
157 decoded > 0x10FFFF ||
158 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
159 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
160 (decoded & 0xFFFE) == 0xFFFE) { // noncharacters
161 input_error(0, 0, "invalid byte sequence in input");
163 } else if ((*src & 0xF0) == 0xE0) {
165 decoded = *src++ & 0x0F;
168 input->utf8_part_decoded_min_code = min_code;
169 input->utf8_part_decoded_char = decoded;
170 input->utf8_part_decoded_rest_len = 2;
173 if ((*src & 0xC0) == 0x80) {
174 decoded = (decoded << 6) | (*src++ & 0x3F);
179 } else if ((*src & 0xF8) == 0xF0) {
181 decoded = *src++ & 0x07;
184 input->utf8_part_decoded_min_code = min_code;
185 input->utf8_part_decoded_char = decoded;
186 input->utf8_part_decoded_rest_len = 3;
189 if ((*src & 0xC0) == 0x80) {
190 decoded = (decoded << 6) | (*src++ & 0x3F);
197 input_error(0, 0, "invalid byte sequence in input");
202 input->utf8_part_decoded_rest_len = 4;
205 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
211 /* we're done when we could read more than 1 char */
217 static size_t decode_windows_1252(input_t *input, utf32 *buffer,
220 unsigned char read_buf[buffer_size];
221 size_t const s = read_block(input, read_buf, sizeof(read_buf));
223 unsigned char const *src = read_buf;
224 unsigned char const *end = read_buf + s;
229 case 0x80: tc = 0x20AC; break; // €
230 case 0x82: tc = 0x201A; break; // ‚
231 case 0x83: tc = 0x0192; break; // ƒ
232 case 0x84: tc = 0x201E; break; // „
233 case 0x85: tc = 0x2026; break; // …
234 case 0x86: tc = 0x2020; break; // †
235 case 0x87: tc = 0x2021; break; // ‡
236 case 0x88: tc = 0x02C6; break; // ˆ
237 case 0x89: tc = 0x2030; break; // ‰
238 case 0x8A: tc = 0x0160; break; // Š
239 case 0x8B: tc = 0x2039; break; // ‹
240 case 0x8C: tc = 0x0152; break; // Œ
241 case 0x8E: tc = 0x017D; break; // Ž
242 case 0x91: tc = 0x2018; break; // ‘
243 case 0x92: tc = 0x2019; break; // ’
244 case 0x93: tc = 0x201C; break; // “
245 case 0x94: tc = 0x201D; break; // ”
246 case 0x95: tc = 0x2022; break; // •
247 case 0x96: tc = 0x2013; break; // –
248 case 0x97: tc = 0x2014; break; // —
249 case 0x98: tc = 0x02DC; break; // ˜
250 case 0x99: tc = 0x2122; break; // ™
251 case 0x9A: tc = 0x0161; break; // š
252 case 0x9B: tc = 0x203A; break; // ›
253 case 0x9C: tc = 0x0153; break; // œ
254 case 0x9E: tc = 0x017E; break; // ž
255 case 0x9F: tc = 0x0178; break; // Ÿ
263 typedef struct named_decoder_t {
268 static named_decoder_t const decoders[] = {
269 { "CP819", decode_iso_8859_1 }, // official alias
270 { "IBM819", decode_iso_8859_1 }, // official alias
271 { "ISO-8859-1", decode_iso_8859_1 }, // official alias
272 { "ISO-8859-15", decode_iso_8859_15 }, // official name
273 { "ISO8859-1", decode_iso_8859_1 },
274 { "ISO8859-15", decode_iso_8859_15 },
275 { "ISO_8859-1", decode_iso_8859_1 }, // official alias
276 { "ISO_8859-15", decode_iso_8859_15 }, // official alias
277 { "ISO_8859-1:1987", decode_iso_8859_1 }, // official name
278 { "Latin-9", decode_iso_8859_15 }, // official alias
279 { "UTF-8", decode_utf8 }, // official name
280 { "csISOLatin1", decode_iso_8859_1 }, // official alias
281 { "cp1252", decode_windows_1252 },
282 { "iso-ir-100", decode_iso_8859_1 }, // official alias
283 { "l1", decode_iso_8859_1 }, // official alias
284 { "latin1", decode_iso_8859_1 }, // official alias
285 { "windows-1252", decode_windows_1252 }, // official name
290 /** strcasecmp is not part of C99 so we need our own implementation here */
291 static int my_strcasecmp(const char *s1, const char *s2)
293 for ( ; *s1 != 0; ++s1, ++s2) {
294 if (tolower(*s1) != tolower(*s2))
297 return (unsigned char)*s1 - (unsigned char)*s2;
300 static void choose_decoder(input_t *result, const char *encoding)
303 for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
304 if (my_strcasecmp(encoding, i->name) != 0)
306 result->decode = i->decoder;
309 errorf(NULL, "input encoding \"%s\" not supported", encoding);
311 result->decode = decode_utf8;
314 input_t *input_from_stream(FILE *file, const char *encoding)
316 input_t *result = XMALLOCZ(input_t);
317 result->kind = INPUT_FILE;
318 result->in.file = file;
320 choose_decoder(result, encoding);
325 input_t *input_from_string(const char *string, const char *encoding)
327 input_t *result = XMALLOCZ(input_t);
328 result->kind = INPUT_STRING;
329 result->in.string = string;
331 choose_decoder(result, encoding);
336 size_t decode(input_t *input, utf32 *buffer, size_t buffer_size)
338 return input->decode(input, buffer, buffer_size);
341 void input_free(input_t *input)