0
|
1 /*
|
|
2 * This code is derivative of guess.c of Gauche-0.8.3.
|
|
3 * The following is the original copyright notice.
|
|
4 */
|
|
5
|
|
6 /*
|
|
7 * guess.c - guessing character encoding
|
|
8 *
|
|
9 * Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
|
|
10 *
|
|
11 * Redistribution and use in source and binary forms, with or without
|
|
12 * modification, are permitted provided that the following conditions
|
|
13 * are met:
|
|
14 *
|
|
15 * 1. Redistributions of source code must retain the above copyright
|
|
16 * notice, this list of conditions and the following disclaimer.
|
|
17 *
|
|
18 * 2. Redistributions in binary form must reproduce the above copyright
|
|
19 * notice, this list of conditions and the following disclaimer in the
|
|
20 * documentation and/or other materials provided with the distribution.
|
|
21 *
|
|
22 * 3. Neither the name of the authors nor the names of its contributors
|
|
23 * may be used to endorse or promote products derived from this
|
|
24 * software without specific prior written permission.
|
|
25 *
|
|
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
27 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
29 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
30 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
31 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
|
32 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
33 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
34 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
35 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
36 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
37 *
|
|
38 */
|
|
39
|
|
40 #include "libguess.h"
|
|
41
|
|
42 /* take precedence if scores are same. you can customize the order as: */
|
|
43 /* ORDER_** &highest, &second, ... &lowest */
|
|
44 #define ORDER_JP &utf8, &sjis, &eucj
|
|
45 #define ORDER_TW &utf8, &big5
|
|
46 #define ORDER_CN &utf8, &gb2312, &gb18030
|
|
47 #define ORDER_KR &utf8, &euck, &johab
|
|
48
|
|
49 /* workaround for that glib's g_convert can't convert
|
|
50 properly from UCS-2BE/LE trailing after BOM. */
|
|
51 #define WITH_G_CONVERT 1
|
|
52 /* #undef WITH_G_CONVERT */
|
|
53
|
|
54 #ifdef WITH_G_CONVERT
|
|
55 const char UCS_2BE[] = "UTF-16";
|
|
56 const char UCS_2LE[] = "UTF-16";
|
|
57 #else
|
|
58 const char UCS_2BE[] = "UCS-2BE";
|
|
59 const char UCS_2LE[] = "UCS-2LE";
|
|
60 #endif
|
|
61
|
|
62 /* data types */
|
|
63 typedef struct guess_arc_rec
|
|
64 {
|
|
65 unsigned int next; /* next state */
|
|
66 double score; /* score */
|
|
67 } guess_arc;
|
|
68
|
|
69 typedef struct guess_dfa_rec
|
|
70 {
|
|
71 signed char (*states)[256];
|
|
72 guess_arc *arcs;
|
|
73 int state;
|
|
74 double score;
|
|
75 } guess_dfa;
|
|
76
|
|
77 /* macros */
|
|
78 #define DFA_INIT(st, ar) \
|
|
79 { st, ar, 0, 1.0 }
|
|
80
|
|
81 #define DFA_NEXT(dfa, ch) \
|
|
82 do { \
|
|
83 int arc__; \
|
|
84 if (dfa.state >= 0) { \
|
|
85 arc__ = dfa.states[dfa.state][ch]; \
|
|
86 if (arc__ < 0) { \
|
|
87 dfa.state = -1; \
|
|
88 } else { \
|
|
89 dfa.state = dfa.arcs[arc__].next; \
|
|
90 dfa.score *= dfa.arcs[arc__].score; \
|
|
91 } \
|
|
92 } \
|
|
93 } while (0)
|
|
94
|
|
95 #define DFA_ALIVE(dfa) (dfa.state >= 0)
|
|
96
|
|
97 /* include DFA table generated by guess.scm */
|
|
98 #include "guess_tab.c"
|
|
99
|
|
100
|
|
101 int dfa_validate_utf8(const char *buf, int buflen)
|
|
102 {
|
|
103 int i;
|
|
104 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
105
|
|
106 for (i = 0; i < buflen; i++) {
|
|
107 int c = (unsigned char) buf[i];
|
|
108
|
|
109 if (DFA_ALIVE(utf8))
|
|
110 DFA_NEXT(utf8, c);
|
|
111 else
|
|
112 break;
|
|
113 }
|
|
114
|
|
115 DFA_NEXT(utf8, '\0'); //Bug #53
|
|
116
|
|
117 if(DFA_ALIVE(utf8))
|
|
118 return 1;
|
|
119 else
|
|
120 return 0;
|
|
121 }
|
|
122
|
|
123 const char *guess_jp(const char *buf, int buflen)
|
|
124 {
|
|
125 int i;
|
|
126 guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
|
|
127 guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
|
|
128 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
129 guess_dfa *top = NULL;
|
|
130
|
|
131 guess_dfa *order[] = { ORDER_JP, NULL };
|
|
132
|
|
133 for (i = 0; i < buflen; i++) {
|
|
134 int c = (unsigned char) buf[i];
|
|
135
|
|
136 /* special treatment of iso-2022 escape sequence */
|
|
137 if (c == 0x1b) {
|
|
138 if (i < buflen - 1) {
|
|
139 c = (unsigned char) buf[++i];
|
|
140 if (c == '$' || c == '(')
|
|
141 return "ISO-2022-JP";
|
|
142 }
|
|
143 }
|
|
144
|
|
145 /* special treatment of BOM */
|
|
146 if (i == 0 && c == 0xff) {
|
|
147 if (i < buflen - 1) {
|
|
148 c = (unsigned char) buf[i + 1];
|
|
149 if (c == 0xfe)
|
|
150 return UCS_2LE;
|
|
151 }
|
|
152 }
|
|
153 if (i == 0 && c == 0xfe) {
|
|
154 if (i < buflen - 1) {
|
|
155 c = (unsigned char) buf[i + 1];
|
|
156 if (c == 0xff)
|
|
157 return UCS_2BE;
|
|
158 }
|
|
159 }
|
|
160
|
|
161 if (DFA_ALIVE(eucj)) {
|
|
162 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8))
|
|
163 return "EUC-JP";
|
|
164 DFA_NEXT(eucj, c);
|
|
165 }
|
|
166 if (DFA_ALIVE(sjis)) {
|
|
167 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8))
|
|
168 return "SJIS";
|
|
169 DFA_NEXT(sjis, c);
|
|
170 }
|
|
171 if (DFA_ALIVE(utf8)) {
|
|
172 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj))
|
|
173 return "UTF-8";
|
|
174 DFA_NEXT(utf8, c);
|
|
175 }
|
|
176
|
|
177 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
|
|
178 /* we ran out the possibilities */
|
|
179 return NULL;
|
|
180 }
|
|
181 }
|
|
182
|
|
183 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
184 one candidate tie, pick the default encoding. */
|
|
185 for (i = 0; order[i] != NULL; i++) {
|
|
186 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
187 if (top == NULL || order[i]->score > top->score)
|
|
188 top = order[i];
|
|
189 }
|
|
190 }
|
|
191
|
|
192 if (top == &eucj)
|
|
193 return "EUC-JP";
|
|
194 if (top == &utf8)
|
|
195 return "UTF-8";
|
|
196 if (top == &sjis)
|
|
197 return "SJIS";
|
|
198 return NULL;
|
|
199 }
|
|
200
|
|
201 const char *guess_tw(const char *buf, int buflen)
|
|
202 {
|
|
203 int i;
|
|
204 guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
|
|
205 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
206 guess_dfa *top = NULL;
|
|
207
|
|
208 guess_dfa *order[] = { ORDER_TW, NULL };
|
|
209
|
|
210 for (i = 0; i < buflen; i++) {
|
|
211 int c = (unsigned char) buf[i];
|
|
212
|
|
213 /* special treatment of iso-2022 escape sequence */
|
|
214 if (c == 0x1b) {
|
|
215 if (i < buflen - 1) {
|
|
216 c = (unsigned char) buf[++i];
|
|
217 if (c == '$' || c == '(')
|
|
218 return "ISO-2022-TW";
|
|
219 }
|
|
220 }
|
|
221
|
|
222 /* special treatment of BOM */
|
|
223 if (i == 0 && c == 0xff) {
|
|
224 if (i < buflen - 1) {
|
|
225 c = (unsigned char) buf[i + 1];
|
|
226 if (c == 0xfe)
|
|
227 return UCS_2LE;
|
|
228 }
|
|
229 }
|
|
230 if (i == 0 && c == 0xfe) {
|
|
231 if (i < buflen - 1) {
|
|
232 c = (unsigned char) buf[i + 1];
|
|
233 if (c == 0xff)
|
|
234 return UCS_2BE;
|
|
235 }
|
|
236 }
|
|
237
|
|
238 if (DFA_ALIVE(big5)) {
|
|
239 if (!DFA_ALIVE(utf8))
|
|
240 return "BIG5";
|
|
241 DFA_NEXT(big5, c);
|
|
242 }
|
|
243 if (DFA_ALIVE(utf8)) {
|
|
244 if (!DFA_ALIVE(big5))
|
|
245 return "UTF-8";
|
|
246 DFA_NEXT(utf8, c);
|
|
247 }
|
|
248
|
|
249 if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
|
|
250 /* we ran out the possibilities */
|
|
251 return NULL;
|
|
252 }
|
|
253 }
|
|
254
|
|
255 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
256 one candidate tie, pick the default encoding. */
|
|
257 for (i = 0; order[i] != NULL; i++) {
|
|
258 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
259 if (top == NULL || order[i]->score > top->score)
|
|
260 top = order[i];
|
|
261 }
|
|
262 }
|
|
263
|
|
264 if (top == &big5)
|
|
265 return "BIG5";
|
|
266 if (top == &utf8)
|
|
267 return "UTF-8";
|
|
268 return NULL;
|
|
269 }
|
|
270
|
|
271 const char *guess_cn(const char *buf, int buflen)
|
|
272 {
|
|
273 int i;
|
|
274 guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
|
|
275 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
276 guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
|
|
277 guess_dfa *top = NULL;
|
|
278
|
|
279 guess_dfa *order[] = { ORDER_CN, NULL };
|
|
280
|
|
281 for (i = 0; i < buflen; i++) {
|
|
282 int c = (unsigned char) buf[i];
|
|
283 int c2;
|
|
284
|
|
285 /* special treatment of iso-2022 escape sequence */
|
|
286 if (c == 0x1b) {
|
|
287 if (i < buflen - 1) {
|
|
288 c = (unsigned char) buf[i + 1];
|
|
289 c2 = (unsigned char) buf[i + 2];
|
|
290 if (c == '$' && (c2 == ')' || c2 == '+'))
|
|
291 return "ISO-2022-CN";
|
|
292 }
|
|
293 }
|
|
294
|
|
295 /* special treatment of BOM */
|
|
296 if (i == 0 && c == 0xff) {
|
|
297 if (i < buflen - 1) {
|
|
298 c = (unsigned char) buf[i + 1];
|
|
299 if (c == 0xfe)
|
|
300 return UCS_2LE;
|
|
301 }
|
|
302 }
|
|
303 if (i == 0 && c == 0xfe) {
|
|
304 if (i < buflen - 1) {
|
|
305 c = (unsigned char) buf[i + 1];
|
|
306 if (c == 0xff)
|
|
307 return UCS_2BE;
|
|
308 }
|
|
309 }
|
|
310
|
|
311 if (DFA_ALIVE(gb2312)) {
|
|
312 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030))
|
|
313 return "GB2312";
|
|
314 DFA_NEXT(gb2312, c);
|
|
315 }
|
|
316 if (DFA_ALIVE(utf8)) {
|
|
317 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030))
|
|
318 return "UTF-8";
|
|
319 DFA_NEXT(utf8, c);
|
|
320 }
|
|
321 if (DFA_ALIVE(gb18030)) {
|
|
322 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312))
|
|
323 return "GB18030";
|
|
324 DFA_NEXT(gb18030, c);
|
|
325 }
|
|
326
|
|
327 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
|
|
328 /* we ran out the possibilities */
|
|
329 return NULL;
|
|
330 }
|
|
331 }
|
|
332
|
|
333 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
334 one candidate tie, pick the default encoding. */
|
|
335 for (i = 0; order[i] != NULL; i++) {
|
|
336 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
337 if (top == NULL || order[i]->score > top->score)
|
|
338 top = order[i];
|
|
339 }
|
|
340 }
|
|
341
|
|
342 if (top == &gb2312)
|
|
343 return "GB2312";
|
|
344 if (top == &utf8)
|
|
345 return "UTF-8";
|
|
346 if (top == &gb18030)
|
|
347 return "GB18030";
|
|
348 return NULL;
|
|
349 }
|
|
350
|
|
351 const char *guess_kr(const char *buf, int buflen)
|
|
352 {
|
|
353 int i;
|
|
354 guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
|
|
355 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
356 guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
|
|
357 guess_dfa *top = NULL;
|
|
358
|
|
359 guess_dfa *order[] = { ORDER_KR, NULL };
|
|
360
|
|
361 for (i = 0; i < buflen; i++) {
|
|
362 int c = (unsigned char) buf[i];
|
|
363 int c2;
|
|
364
|
|
365 /* special treatment of iso-2022 escape sequence */
|
|
366 if (c == 0x1b) {
|
|
367 if (i < buflen - 1) {
|
|
368 c = (unsigned char) buf[i + 1];
|
|
369 c2 = (unsigned char) buf[i + 2];
|
|
370 if (c == '$' && c2 == ')')
|
|
371 return "ISO-2022-KR";
|
|
372 }
|
|
373 }
|
|
374
|
|
375 /* special treatment of BOM */
|
|
376 if (i == 0 && c == 0xff) {
|
|
377 if (i < buflen - 1) {
|
|
378 c = (unsigned char) buf[i + 1];
|
|
379 if (c == 0xfe)
|
|
380 return UCS_2LE;
|
|
381 }
|
|
382 }
|
|
383 if (i == 0 && c == 0xfe) {
|
|
384 if (i < buflen - 1) {
|
|
385 c = (unsigned char) buf[i + 1];
|
|
386 if (c == 0xff)
|
|
387 return UCS_2BE;
|
|
388 }
|
|
389 }
|
|
390
|
|
391 if (DFA_ALIVE(euck)) {
|
|
392 if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8))
|
|
393 return "EUC-KR";
|
|
394 DFA_NEXT(euck, c);
|
|
395 }
|
|
396 if (DFA_ALIVE(johab)) {
|
|
397 if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8))
|
|
398 return "JOHAB";
|
|
399 DFA_NEXT(johab, c);
|
|
400 }
|
|
401 if (DFA_ALIVE(utf8)) {
|
|
402 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab))
|
|
403 return "UTF-8";
|
|
404 DFA_NEXT(utf8, c);
|
|
405 }
|
|
406
|
|
407 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
|
|
408 /* we ran out the possibilities */
|
|
409 return NULL;
|
|
410 }
|
|
411 }
|
|
412
|
|
413 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
414 one candidate tie, pick the default encoding. */
|
|
415 for (i = 0; order[i] != NULL; i++) {
|
|
416 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
417 if (top == NULL || order[i]->score > top->score)
|
|
418 top = order[i];
|
|
419 }
|
|
420 }
|
|
421
|
|
422 if (top == &euck)
|
|
423 return "EUC-KR";
|
|
424 if (top == &utf8)
|
|
425 return "UTF-8";
|
|
426 if (top == &johab)
|
|
427 return "JOHAB";
|
|
428 return NULL;
|
|
429 }
|