Mercurial > libguess

comparison cjk_impl.c @ 0:d9b6ff839eab

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
initial import
author Yoshiki Yazawa <yaz@cc.rim.or.jp>
date Fri, 30 Nov 2007 19:34:51 +0900
parents
children 754a4550c64e
comparison
equal deleted inserted replaced
-1:000000000000 0:d9b6ff839eab
1 /*
2 * This code is derivative of guess.c of Gauche-0.8.3.
3 * The following is the original copyright notice.
4 */
5
6 /*
7 * guess.c - guessing character encoding
8 *
9 * Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 *
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 *
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * 3. Neither the name of the authors nor the names of its contributors
23 * may be used to endorse or promote products derived from this
24 * software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
30 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
32 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 *
38 */
39
40 #include "libguess.h"
41
42 /* take precedence if scores are same. you can customize the order as: */
43 /* ORDER_** &highest, &second, ... &lowest */
44 #define ORDER_JP &utf8, &sjis, &eucj
45 #define ORDER_TW &utf8, &big5
46 #define ORDER_CN &utf8, &gb2312, &gb18030
47 #define ORDER_KR &utf8, &euck, &johab
48
49 /* workaround for that glib's g_convert can't convert
50 properly from UCS-2BE/LE trailing after BOM. */
51 #define WITH_G_CONVERT 1
52 /* #undef WITH_G_CONVERT */
53
54 #ifdef WITH_G_CONVERT
55 const char UCS_2BE[] = "UTF-16";
56 const char UCS_2LE[] = "UTF-16";
57 #else
58 const char UCS_2BE[] = "UCS-2BE";
59 const char UCS_2LE[] = "UCS-2LE";
60 #endif
61
62 /* data types */
63 typedef struct guess_arc_rec
64 {
65 unsigned int next; /* next state */
66 double score; /* score */
67 } guess_arc;
68
69 typedef struct guess_dfa_rec
70 {
71 signed char (*states)[256];
72 guess_arc *arcs;
73 int state;
74 double score;
75 } guess_dfa;
76
77 /* macros */
78 #define DFA_INIT(st, ar) \
79 { st, ar, 0, 1.0 }
80
81 #define DFA_NEXT(dfa, ch) \
82 do { \
83 int arc__; \
84 if (dfa.state >= 0) { \
85 arc__ = dfa.states[dfa.state][ch]; \
86 if (arc__ < 0) { \
87 dfa.state = -1; \
88 } else { \
89 dfa.state = dfa.arcs[arc__].next; \
90 dfa.score *= dfa.arcs[arc__].score; \
91 } \
92 } \
93 } while (0)
94
95 #define DFA_ALIVE(dfa) (dfa.state >= 0)
96
97 /* include DFA table generated by guess.scm */
98 #include "guess_tab.c"
99
100
101 int dfa_validate_utf8(const char *buf, int buflen)
102 {
103 int i;
104 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
105
106 for (i = 0; i < buflen; i++) {
107 int c = (unsigned char) buf[i];
108
109 if (DFA_ALIVE(utf8))
110 DFA_NEXT(utf8, c);
111 else
112 break;
113 }
114
115 DFA_NEXT(utf8, '\0'); //Bug #53
116
117 if(DFA_ALIVE(utf8))
118 return 1;
119 else
120 return 0;
121 }
122
123 const char *guess_jp(const char *buf, int buflen)
124 {
125 int i;
126 guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
127 guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
128 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
129 guess_dfa *top = NULL;
130
131 guess_dfa *order[] = { ORDER_JP, NULL };
132
133 for (i = 0; i < buflen; i++) {
134 int c = (unsigned char) buf[i];
135
136 /* special treatment of iso-2022 escape sequence */
137 if (c == 0x1b) {
138 if (i < buflen - 1) {
139 c = (unsigned char) buf[++i];
140 if (c == '$' || c == '(')
141 return "ISO-2022-JP";
142 }
143 }
144
145 /* special treatment of BOM */
146 if (i == 0 && c == 0xff) {
147 if (i < buflen - 1) {
148 c = (unsigned char) buf[i + 1];
149 if (c == 0xfe)
150 return UCS_2LE;
151 }
152 }
153 if (i == 0 && c == 0xfe) {
154 if (i < buflen - 1) {
155 c = (unsigned char) buf[i + 1];
156 if (c == 0xff)
157 return UCS_2BE;
158 }
159 }
160
161 if (DFA_ALIVE(eucj)) {
162 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8))
163 return "EUC-JP";
164 DFA_NEXT(eucj, c);
165 }
166 if (DFA_ALIVE(sjis)) {
167 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8))
168 return "SJIS";
169 DFA_NEXT(sjis, c);
170 }
171 if (DFA_ALIVE(utf8)) {
172 if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj))
173 return "UTF-8";
174 DFA_NEXT(utf8, c);
175 }
176
177 if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
178 /* we ran out the possibilities */
179 return NULL;
180 }
181 }
182
183 /* Now, we have ambigous code. Pick the highest score. If more than
184 one candidate tie, pick the default encoding. */
185 for (i = 0; order[i] != NULL; i++) {
186 if (order[i]->state >= 0) { //DFA_ALIVE()
187 if (top == NULL || order[i]->score > top->score)
188 top = order[i];
189 }
190 }
191
192 if (top == &eucj)
193 return "EUC-JP";
194 if (top == &utf8)
195 return "UTF-8";
196 if (top == &sjis)
197 return "SJIS";
198 return NULL;
199 }
200
201 const char *guess_tw(const char *buf, int buflen)
202 {
203 int i;
204 guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
205 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
206 guess_dfa *top = NULL;
207
208 guess_dfa *order[] = { ORDER_TW, NULL };
209
210 for (i = 0; i < buflen; i++) {
211 int c = (unsigned char) buf[i];
212
213 /* special treatment of iso-2022 escape sequence */
214 if (c == 0x1b) {
215 if (i < buflen - 1) {
216 c = (unsigned char) buf[++i];
217 if (c == '$' || c == '(')
218 return "ISO-2022-TW";
219 }
220 }
221
222 /* special treatment of BOM */
223 if (i == 0 && c == 0xff) {
224 if (i < buflen - 1) {
225 c = (unsigned char) buf[i + 1];
226 if (c == 0xfe)
227 return UCS_2LE;
228 }
229 }
230 if (i == 0 && c == 0xfe) {
231 if (i < buflen - 1) {
232 c = (unsigned char) buf[i + 1];
233 if (c == 0xff)
234 return UCS_2BE;
235 }
236 }
237
238 if (DFA_ALIVE(big5)) {
239 if (!DFA_ALIVE(utf8))
240 return "BIG5";
241 DFA_NEXT(big5, c);
242 }
243 if (DFA_ALIVE(utf8)) {
244 if (!DFA_ALIVE(big5))
245 return "UTF-8";
246 DFA_NEXT(utf8, c);
247 }
248
249 if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
250 /* we ran out the possibilities */
251 return NULL;
252 }
253 }
254
255 /* Now, we have ambigous code. Pick the highest score. If more than
256 one candidate tie, pick the default encoding. */
257 for (i = 0; order[i] != NULL; i++) {
258 if (order[i]->state >= 0) { //DFA_ALIVE()
259 if (top == NULL || order[i]->score > top->score)
260 top = order[i];
261 }
262 }
263
264 if (top == &big5)
265 return "BIG5";
266 if (top == &utf8)
267 return "UTF-8";
268 return NULL;
269 }
270
271 const char *guess_cn(const char *buf, int buflen)
272 {
273 int i;
274 guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
275 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
276 guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
277 guess_dfa *top = NULL;
278
279 guess_dfa *order[] = { ORDER_CN, NULL };
280
281 for (i = 0; i < buflen; i++) {
282 int c = (unsigned char) buf[i];
283 int c2;
284
285 /* special treatment of iso-2022 escape sequence */
286 if (c == 0x1b) {
287 if (i < buflen - 1) {
288 c = (unsigned char) buf[i + 1];
289 c2 = (unsigned char) buf[i + 2];
290 if (c == '$' && (c2 == ')' || c2 == '+'))
291 return "ISO-2022-CN";
292 }
293 }
294
295 /* special treatment of BOM */
296 if (i == 0 && c == 0xff) {
297 if (i < buflen - 1) {
298 c = (unsigned char) buf[i + 1];
299 if (c == 0xfe)
300 return UCS_2LE;
301 }
302 }
303 if (i == 0 && c == 0xfe) {
304 if (i < buflen - 1) {
305 c = (unsigned char) buf[i + 1];
306 if (c == 0xff)
307 return UCS_2BE;
308 }
309 }
310
311 if (DFA_ALIVE(gb2312)) {
312 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030))
313 return "GB2312";
314 DFA_NEXT(gb2312, c);
315 }
316 if (DFA_ALIVE(utf8)) {
317 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030))
318 return "UTF-8";
319 DFA_NEXT(utf8, c);
320 }
321 if (DFA_ALIVE(gb18030)) {
322 if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312))
323 return "GB18030";
324 DFA_NEXT(gb18030, c);
325 }
326
327 if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
328 /* we ran out the possibilities */
329 return NULL;
330 }
331 }
332
333 /* Now, we have ambigous code. Pick the highest score. If more than
334 one candidate tie, pick the default encoding. */
335 for (i = 0; order[i] != NULL; i++) {
336 if (order[i]->state >= 0) { //DFA_ALIVE()
337 if (top == NULL || order[i]->score > top->score)
338 top = order[i];
339 }
340 }
341
342 if (top == &gb2312)
343 return "GB2312";
344 if (top == &utf8)
345 return "UTF-8";
346 if (top == &gb18030)
347 return "GB18030";
348 return NULL;
349 }
350
351 const char *guess_kr(const char *buf, int buflen)
352 {
353 int i;
354 guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
355 guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
356 guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
357 guess_dfa *top = NULL;
358
359 guess_dfa *order[] = { ORDER_KR, NULL };
360
361 for (i = 0; i < buflen; i++) {
362 int c = (unsigned char) buf[i];
363 int c2;
364
365 /* special treatment of iso-2022 escape sequence */
366 if (c == 0x1b) {
367 if (i < buflen - 1) {
368 c = (unsigned char) buf[i + 1];
369 c2 = (unsigned char) buf[i + 2];
370 if (c == '$' && c2 == ')')
371 return "ISO-2022-KR";
372 }
373 }
374
375 /* special treatment of BOM */
376 if (i == 0 && c == 0xff) {
377 if (i < buflen - 1) {
378 c = (unsigned char) buf[i + 1];
379 if (c == 0xfe)
380 return UCS_2LE;
381 }
382 }
383 if (i == 0 && c == 0xfe) {
384 if (i < buflen - 1) {
385 c = (unsigned char) buf[i + 1];
386 if (c == 0xff)
387 return UCS_2BE;
388 }
389 }
390
391 if (DFA_ALIVE(euck)) {
392 if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8))
393 return "EUC-KR";
394 DFA_NEXT(euck, c);
395 }
396 if (DFA_ALIVE(johab)) {
397 if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8))
398 return "JOHAB";
399 DFA_NEXT(johab, c);
400 }
401 if (DFA_ALIVE(utf8)) {
402 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab))
403 return "UTF-8";
404 DFA_NEXT(utf8, c);
405 }
406
407 if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
408 /* we ran out the possibilities */
409 return NULL;
410 }
411 }
412
413 /* Now, we have ambigous code. Pick the highest score. If more than
414 one candidate tie, pick the default encoding. */
415 for (i = 0; order[i] != NULL; i++) {
416 if (order[i]->state >= 0) { //DFA_ALIVE()
417 if (top == NULL || order[i]->score > top->score)
418 top = order[i];
419 }
420 }
421
422 if (top == &euck)
423 return "EUC-KR";
424 if (top == &utf8)
425 return "UTF-8";
426 if (top == &johab)
427 return "JOHAB";
428 return NULL;
429 }