2313
|
1 /*
|
|
2 * This code is derivative of guess.c of Gauche-0.8.3.
|
|
3 * The following is the original copyright notice.
|
|
4 */
|
|
5
|
|
6 /*
|
|
7 * guess.c - guessing character encoding
|
|
8 *
|
|
9 * Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
|
|
10 *
|
|
11 * Redistribution and use in source and binary forms, with or without
|
|
12 * modification, are permitted provided that the following conditions
|
|
13 * are met:
|
|
14 *
|
|
15 * 1. Redistributions of source code must retain the above copyright
|
|
16 * notice, this list of conditions and the following disclaimer.
|
|
17 *
|
|
18 * 2. Redistributions in binary form must reproduce the above copyright
|
|
19 * notice, this list of conditions and the following disclaimer in the
|
|
20 * documentation and/or other materials provided with the distribution.
|
|
21 *
|
|
22 * 3. Neither the name of the authors nor the names of its contributors
|
|
23 * may be used to endorse or promote products derived from this
|
|
24 * software without specific prior written permission.
|
|
25 *
|
|
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
27 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
29 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
30 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
31 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
|
32 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
33 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
34 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
35 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
36 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
37 *
|
|
38 */
|
|
39
|
|
40 #include "libguess.h"
|
|
41 #define NULL ((void *)0)
|
|
42
|
|
43 /* take precedence if scores are same. you can customize the order as: */
|
|
44 /* ORDER_** &highest, &second, ... &lowest */
|
|
45 #define ORDER_JP &utf8, &sjis, &eucj
|
|
46 #define ORDER_TW &utf8, &big5
|
|
47 #define ORDER_CN &utf8, &gb2312, &gb18030
|
|
48 #define ORDER_KR &utf8, &euck, &johab
|
|
49
|
|
50 /* workaround for that glib's g_convert can't convert
|
|
51 properly from UCS-2BE/LE trailing after BOM. */
|
|
52 #define WITH_G_CONVERT 1
|
|
53 /* #undef WITH_G_CONVERT */
|
|
54
|
|
55 #ifdef WITH_G_CONVERT
|
|
56 const char UCS_2BE[] = "UTF-16";
|
|
57 const char UCS_2LE[] = "UTF-16";
|
|
58 #else
|
|
59 const char UCS_2BE[] = "UCS-2BE";
|
|
60 const char UCS_2LE[] = "UCS-2LE";
|
|
61 #endif
|
|
62
|
|
63 /* data types */
|
|
64 typedef struct guess_arc_rec
|
|
65 {
|
|
66 unsigned int next; /* next state */
|
|
67 double score; /* score */
|
|
68 } guess_arc;
|
|
69
|
|
70 typedef struct guess_dfa_rec
|
|
71 {
|
|
72 signed char (*states)[256];
|
|
73 guess_arc *arcs;
|
|
74 int state;
|
|
75 double score;
|
|
76 } guess_dfa;
|
|
77
|
|
78 /* macros */
|
|
79 #define DFA_INIT(st, ar) \
|
|
80 { st, ar, 0, 1.0 }
|
|
81
|
|
82 #define DFA_NEXT(dfa, ch) \
|
|
83 do { \
|
|
84 int arc__; \
|
|
85 if (dfa.state >= 0) { \
|
|
86 arc__ = dfa.states[dfa.state][ch]; \
|
|
87 if (arc__ < 0) { \
|
|
88 dfa.state = -1; \
|
|
89 } else { \
|
|
90 dfa.state = dfa.arcs[arc__].next; \
|
|
91 dfa.score *= dfa.arcs[arc__].score; \
|
|
92 } \
|
|
93 } \
|
|
94 } while (0)
|
|
95
|
|
96 #define DFA_ALIVE(dfa) (dfa.state >= 0)
|
|
97
|
|
98 /* include DFA table generated by guess.scm */
|
|
99 #include "guess_tab.c"
|
|
100
|
2559
|
101
|
|
102 int dfa_validate_utf8(const char *buf, int buflen)
|
|
103 {
|
|
104 int i;
|
|
105 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
106
|
|
107 for (i = 0; i < buflen; i++) {
|
|
108 int c = (unsigned char) buf[i];
|
|
109
|
|
110 if (DFA_ALIVE(utf8))
|
|
111 DFA_NEXT(utf8, c);
|
|
112 else
|
|
113 return 0;
|
|
114 }
|
|
115 return 1;
|
|
116 }
|
|
117
|
2313
|
118 const char *guess_jp(const char *buf, int buflen)
|
|
119 {
|
|
120 int i;
|
|
121 guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
|
|
122 guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
|
|
123 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
124 guess_dfa *top = NULL;
|
|
125
|
|
126 guess_dfa *order[] = { ORDER_JP, NULL };
|
|
127
|
|
128 for (i = 0; i < buflen; i++) {
|
|
129 int c = (unsigned char) buf[i];
|
|
130
|
|
131 /* special treatment of iso-2022 escape sequence */
|
|
132 if (c == 0x1b) {
|
|
133 if (i < buflen - 1) {
|
|
134 c = (unsigned char) buf[++i];
|
|
135 if (c == '$' || c == '(')
|
|
136 return "ISO-2022-JP";
|
|
137 }
|
|
138 }
|
|
139
|
|
140 /* special treatment of BOM */
|
|
141 if (i == 0 && c == 0xff) {
|
|
142 if (i < buflen - 1) {
|
|
143 c = (unsigned char) buf[i + 1];
|
|
144 if (c == 0xfe)
|
|
145 return UCS_2LE;
|
|
146 }
|
|
147 }
|
|
148 if (i == 0 && c == 0xfe) {
|
|
149 if (i < buflen - 1) {
|
|
150 c = (unsigned char) buf[i + 1];
|
|
151 if (c == 0xff)
|
|
152 return UCS_2BE;
|
|
153 }
|
|
154 }
|
|
155
|
|
156 if (DFA_ALIVE(eucj)) {
|
|
157 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8))
|
|
158 return "EUC-JP";
|
|
159 DFA_NEXT(eucj, c);
|
|
160 }
|
|
161 if (DFA_ALIVE(sjis)) {
|
|
162 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8))
|
|
163 return "SJIS";
|
|
164 DFA_NEXT(sjis, c);
|
|
165 }
|
|
166 if (DFA_ALIVE(utf8)) {
|
|
167 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj))
|
|
168 return "UTF-8";
|
|
169 DFA_NEXT(utf8, c);
|
|
170 }
|
|
171
|
|
172 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
|
|
173 /* we ran out the possibilities */
|
|
174 return NULL;
|
|
175 }
|
|
176 }
|
|
177
|
|
178 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
179 one candidate tie, pick the default encoding. */
|
|
180 for (i = 0; order[i] != NULL; i++) {
|
|
181 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
182 if (top == NULL || order[i]->score > top->score)
|
|
183 top = order[i];
|
|
184 }
|
|
185 }
|
|
186
|
|
187 if (top == &eucj)
|
|
188 return "EUC-JP";
|
|
189 if (top == &utf8)
|
|
190 return "UTF-8";
|
|
191 if (top == &sjis)
|
|
192 return "SJIS";
|
|
193 return NULL;
|
|
194 }
|
|
195
|
|
196 const char *guess_tw(const char *buf, int buflen)
|
|
197 {
|
|
198 int i;
|
|
199 guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
|
|
200 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
201 guess_dfa *top = NULL;
|
|
202
|
|
203 guess_dfa *order[] = { ORDER_TW, NULL };
|
|
204
|
|
205 for (i = 0; i < buflen; i++) {
|
|
206 int c = (unsigned char) buf[i];
|
|
207
|
|
208 /* special treatment of iso-2022 escape sequence */
|
|
209 if (c == 0x1b) {
|
|
210 if (i < buflen - 1) {
|
|
211 c = (unsigned char) buf[++i];
|
|
212 if (c == '$' || c == '(')
|
|
213 return "ISO-2022-TW";
|
|
214 }
|
|
215 }
|
|
216
|
|
217 /* special treatment of BOM */
|
|
218 if (i == 0 && c == 0xff) {
|
|
219 if (i < buflen - 1) {
|
|
220 c = (unsigned char) buf[i + 1];
|
|
221 if (c == 0xfe)
|
|
222 return UCS_2LE;
|
|
223 }
|
|
224 }
|
|
225 if (i == 0 && c == 0xfe) {
|
|
226 if (i < buflen - 1) {
|
|
227 c = (unsigned char) buf[i + 1];
|
|
228 if (c == 0xff)
|
|
229 return UCS_2BE;
|
|
230 }
|
|
231 }
|
|
232
|
|
233 if (DFA_ALIVE(big5)) {
|
|
234 if (!DFA_ALIVE(utf8))
|
|
235 return "BIG5";
|
|
236 DFA_NEXT(big5, c);
|
|
237 }
|
|
238 if (DFA_ALIVE(utf8)) {
|
|
239 if (!DFA_ALIVE(big5))
|
|
240 return "UTF-8";
|
|
241 DFA_NEXT(utf8, c);
|
|
242 }
|
|
243
|
|
244 if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
|
|
245 /* we ran out the possibilities */
|
|
246 return NULL;
|
|
247 }
|
|
248 }
|
|
249
|
|
250 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
251 one candidate tie, pick the default encoding. */
|
|
252 for (i = 0; order[i] != NULL; i++) {
|
|
253 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
254 if (top == NULL || order[i]->score > top->score)
|
|
255 top = order[i];
|
|
256 }
|
|
257 }
|
|
258
|
|
259 if (top == &big5)
|
|
260 return "BIG5";
|
|
261 if (top == &utf8)
|
|
262 return "UTF-8";
|
|
263 return NULL;
|
|
264 }
|
|
265
|
|
266 const char *guess_cn(const char *buf, int buflen)
|
|
267 {
|
|
268 int i;
|
|
269 guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
|
|
270 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
271 guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
|
|
272 guess_dfa *top = NULL;
|
|
273
|
|
274 guess_dfa *order[] = { ORDER_CN, NULL };
|
|
275
|
|
276 for (i = 0; i < buflen; i++) {
|
|
277 int c = (unsigned char) buf[i];
|
|
278 int c2;
|
|
279
|
|
280 /* special treatment of iso-2022 escape sequence */
|
|
281 if (c == 0x1b) {
|
|
282 if (i < buflen - 1) {
|
|
283 c = (unsigned char) buf[i + 1];
|
|
284 c2 = (unsigned char) buf[i + 2];
|
|
285 if (c == '$' && (c2 == ')' || c2 == '+'))
|
|
286 return "ISO-2022-CN";
|
|
287 }
|
|
288 }
|
|
289
|
|
290 /* special treatment of BOM */
|
|
291 if (i == 0 && c == 0xff) {
|
|
292 if (i < buflen - 1) {
|
|
293 c = (unsigned char) buf[i + 1];
|
|
294 if (c == 0xfe)
|
|
295 return UCS_2LE;
|
|
296 }
|
|
297 }
|
|
298 if (i == 0 && c == 0xfe) {
|
|
299 if (i < buflen - 1) {
|
|
300 c = (unsigned char) buf[i + 1];
|
|
301 if (c == 0xff)
|
|
302 return UCS_2BE;
|
|
303 }
|
|
304 }
|
|
305
|
|
306 if (DFA_ALIVE(gb2312)) {
|
|
307 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030))
|
|
308 return "GB2312";
|
|
309 DFA_NEXT(gb2312, c);
|
|
310 }
|
|
311 if (DFA_ALIVE(utf8)) {
|
|
312 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030))
|
|
313 return "UTF-8";
|
|
314 DFA_NEXT(utf8, c);
|
|
315 }
|
|
316 if (DFA_ALIVE(gb18030)) {
|
|
317 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312))
|
|
318 return "GB18030";
|
|
319 DFA_NEXT(gb18030, c);
|
|
320 }
|
|
321
|
|
322 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
|
|
323 /* we ran out the possibilities */
|
|
324 return NULL;
|
|
325 }
|
|
326 }
|
|
327
|
|
328 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
329 one candidate tie, pick the default encoding. */
|
|
330 for (i = 0; order[i] != NULL; i++) {
|
|
331 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
332 if (top == NULL || order[i]->score > top->score)
|
|
333 top = order[i];
|
|
334 }
|
|
335 }
|
|
336
|
|
337 if (top == &gb2312)
|
|
338 return "GB2312";
|
|
339 if (top == &utf8)
|
|
340 return "UTF-8";
|
|
341 if (top == &gb18030)
|
|
342 return "GB18030";
|
|
343 return NULL;
|
|
344 }
|
|
345
|
|
346 const char *guess_kr(const char *buf, int buflen)
|
|
347 {
|
|
348 int i;
|
|
349 guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
|
|
350 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
|
|
351 guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
|
|
352 guess_dfa *top = NULL;
|
|
353
|
|
354 guess_dfa *order[] = { ORDER_KR, NULL };
|
|
355
|
|
356 for (i = 0; i < buflen; i++) {
|
|
357 int c = (unsigned char) buf[i];
|
|
358 int c2;
|
|
359
|
|
360 /* special treatment of iso-2022 escape sequence */
|
|
361 if (c == 0x1b) {
|
|
362 if (i < buflen - 1) {
|
|
363 c = (unsigned char) buf[i + 1];
|
|
364 c2 = (unsigned char) buf[i + 2];
|
|
365 if (c == '$' && c2 == ')')
|
|
366 return "ISO-2022-KR";
|
|
367 }
|
|
368 }
|
|
369
|
|
370 /* special treatment of BOM */
|
|
371 if (i == 0 && c == 0xff) {
|
|
372 if (i < buflen - 1) {
|
|
373 c = (unsigned char) buf[i + 1];
|
|
374 if (c == 0xfe)
|
|
375 return UCS_2LE;
|
|
376 }
|
|
377 }
|
|
378 if (i == 0 && c == 0xfe) {
|
|
379 if (i < buflen - 1) {
|
|
380 c = (unsigned char) buf[i + 1];
|
|
381 if (c == 0xff)
|
|
382 return UCS_2BE;
|
|
383 }
|
|
384 }
|
|
385
|
|
386 if (DFA_ALIVE(euck)) {
|
|
387 if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8))
|
|
388 return "EUC-KR";
|
|
389 DFA_NEXT(euck, c);
|
|
390 }
|
|
391 if (DFA_ALIVE(johab)) {
|
|
392 if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8))
|
|
393 return "JOHAB";
|
|
394 DFA_NEXT(johab, c);
|
|
395 }
|
|
396 if (DFA_ALIVE(utf8)) {
|
|
397 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab))
|
|
398 return "UTF-8";
|
|
399 DFA_NEXT(utf8, c);
|
|
400 }
|
|
401
|
|
402 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
|
|
403 /* we ran out the possibilities */
|
|
404 return NULL;
|
|
405 }
|
|
406 }
|
|
407
|
|
408 /* Now, we have ambigous code. Pick the highest score. If more than
|
|
409 one candidate tie, pick the default encoding. */
|
|
410 for (i = 0; order[i] != NULL; i++) {
|
|
411 if (order[i]->state >= 0) { //DFA_ALIVE()
|
|
412 if (top == NULL || order[i]->score > top->score)
|
|
413 top = order[i];
|
|
414 }
|
|
415 }
|
|
416
|
|
417 if (top == &euck)
|
|
418 return "EUC-KR";
|
|
419 if (top == &utf8)
|
|
420 return "UTF-8";
|
|
421 if (top == &johab)
|
|
422 return "JOHAB";
|
|
423 return NULL;
|
|
424 }
|