changeset 3213:1b9251ab3655 trunk

Split off CJK implementation from guess.c logic.
author William Pitcock <nenolod@atheme-project.org>
date Thu, 02 Aug 2007 01:38:15 -0500
parents 5939941ba48b
children faf6daa29d5c
files src/libguess/Makefile src/libguess/cjk_impl.c src/libguess/guess.c
diffstat 3 files changed, 429 insertions(+), 427 deletions(-) [+]
line wrap: on
line diff
--- a/src/libguess/Makefile	Wed Aug 01 08:13:05 2007 -0500
+++ b/src/libguess/Makefile	Thu Aug 02 01:38:15 2007 -0500
@@ -11,6 +11,7 @@
 SOURCES = \
 	guess.c \
 	arabic_impl.c \
+	cjk_impl.c \
 	russian_impl.c \
 	turkish_impl.c
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/libguess/cjk_impl.c	Thu Aug 02 01:38:15 2007 -0500
@@ -0,0 +1,428 @@
+/*
+ * This code is derivative of guess.c of Gauche-0.8.3.
+ * The following is the original copyright notice.
+ */
+
+/*
+ * guess.c - guessing character encoding 
+ *
+ *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
+ * 
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ * 
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the authors nor the names of its contributors
+ *      may be used to endorse or promote products derived from this
+ *      software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ *   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ *   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ *   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ *   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "libguess.h"
+#define NULL ((void *)0)
+
+/* take precedence if scores are same. you can customize the order as: */
+/* ORDER_** &highest, &second, ... &lowest */
+#define ORDER_JP &utf8, &sjis, &eucj
+#define ORDER_TW &utf8, &big5
+#define ORDER_CN &utf8, &gb2312, &gb18030
+#define ORDER_KR &utf8, &euck, &johab
+
+/* workaround for that glib's g_convert can't convert
+   properly from UCS-2BE/LE trailing after BOM. */
+#define WITH_G_CONVERT 1
+/* #undef WITH_G_CONVERT */
+
+#ifdef WITH_G_CONVERT
+const char UCS_2BE[] = "UTF-16";
+const char UCS_2LE[] = "UTF-16";
+#else
+const char UCS_2BE[] = "UCS-2BE";
+const char UCS_2LE[] = "UCS-2LE";
+#endif
+
+/* data types */
+typedef struct guess_arc_rec
+{
+    unsigned int next;          /* next state */
+    double score;               /* score */
+} guess_arc;
+
+typedef struct guess_dfa_rec
+{
+    signed char (*states)[256];
+    guess_arc *arcs;
+    int state;
+    double score;
+} guess_dfa;
+
+/* macros */
+#define DFA_INIT(st, ar) \
+    { st, ar, 0, 1.0 }
+
+#define DFA_NEXT(dfa, ch)                               \
+    do {                                                \
+        int arc__;                                      \
+        if (dfa.state >= 0) {                           \
+            arc__ = dfa.states[dfa.state][ch];          \
+            if (arc__ < 0) {                            \
+                dfa.state = -1;                         \
+            } else {                                    \
+                dfa.state = dfa.arcs[arc__].next;       \
+                dfa.score *= dfa.arcs[arc__].score;     \
+            }                                           \
+        }                                               \
+    } while (0)
+
+#define DFA_ALIVE(dfa)  (dfa.state >= 0)
+
+/* include DFA table generated by guess.scm */
+#include "guess_tab.c"
+
+
+int dfa_validate_utf8(const char *buf, int buflen)
+{
+    int i;
+    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
+
+    for (i = 0; i < buflen; i++) {
+        int c = (unsigned char) buf[i];
+
+        if (DFA_ALIVE(utf8))
+            DFA_NEXT(utf8, c);
+        else
+            break;
+    }
+
+    if(DFA_ALIVE(utf8))
+        return 1;
+    else 
+        return 0;
+}
+
+const char *guess_jp(const char *buf, int buflen)
+{
+    int i;
+    guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
+    guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
+    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
+    guess_dfa *top = NULL;
+
+    guess_dfa *order[] = { ORDER_JP, NULL };
+
+    for (i = 0; i < buflen; i++) {
+        int c = (unsigned char) buf[i];
+
+        /* special treatment of iso-2022 escape sequence */
+        if (c == 0x1b) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[++i];
+                if (c == '$' || c == '(')
+                    return "ISO-2022-JP";
+            }
+        }
+
+        /* special treatment of BOM */
+        if (i == 0 && c == 0xff) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xfe)
+                    return UCS_2LE;
+            }
+        }
+        if (i == 0 && c == 0xfe) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xff)
+                    return UCS_2BE;
+            }
+        }
+
+        if (DFA_ALIVE(eucj)) {
+            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8))
+                return "EUC-JP";
+            DFA_NEXT(eucj, c);
+        }
+        if (DFA_ALIVE(sjis)) {
+            if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8))
+                return "SJIS";
+            DFA_NEXT(sjis, c);
+        }
+        if (DFA_ALIVE(utf8)) {
+            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj))
+                return "UTF-8";
+            DFA_NEXT(utf8, c);
+        }
+
+        if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
+            /* we ran out the possibilities */
+            return NULL;
+        }
+    }
+
+    /* Now, we have ambigous code.  Pick the highest score.  If more than
+       one candidate tie, pick the default encoding. */
+    for (i = 0; order[i] != NULL; i++) {
+        if (order[i]->state >= 0) { //DFA_ALIVE()
+            if (top == NULL || order[i]->score > top->score)
+                top = order[i];
+        }
+    }
+
+    if (top == &eucj)
+        return "EUC-JP";
+    if (top == &utf8)
+        return "UTF-8";
+    if (top == &sjis)
+        return "SJIS";
+    return NULL;
+}
+
+const char *guess_tw(const char *buf, int buflen)
+{
+    int i;
+    guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
+    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
+    guess_dfa *top = NULL;
+
+    guess_dfa *order[] = { ORDER_TW, NULL };
+
+    for (i = 0; i < buflen; i++) {
+        int c = (unsigned char) buf[i];
+
+        /* special treatment of iso-2022 escape sequence */
+        if (c == 0x1b) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[++i];
+                if (c == '$' || c == '(')
+                    return "ISO-2022-TW";
+            }
+        }
+
+        /* special treatment of BOM */
+        if (i == 0 && c == 0xff) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xfe)
+                    return UCS_2LE;
+            }
+        }
+        if (i == 0 && c == 0xfe) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xff)
+                    return UCS_2BE;
+            }
+        }
+
+        if (DFA_ALIVE(big5)) {
+            if (!DFA_ALIVE(utf8))
+                return "BIG5";
+            DFA_NEXT(big5, c);
+        }
+        if (DFA_ALIVE(utf8)) {
+            if (!DFA_ALIVE(big5))
+                return "UTF-8";
+            DFA_NEXT(utf8, c);
+        }
+
+        if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
+            /* we ran out the possibilities */
+            return NULL;
+        }
+    }
+
+    /* Now, we have ambigous code.  Pick the highest score.  If more than
+       one candidate tie, pick the default encoding. */
+    for (i = 0; order[i] != NULL; i++) {
+        if (order[i]->state >= 0) { //DFA_ALIVE()
+            if (top == NULL || order[i]->score > top->score)
+                top = order[i];
+        }
+    }
+
+    if (top == &big5)
+        return "BIG5";
+    if (top == &utf8)
+        return "UTF-8";
+    return NULL;
+}
+
+const char *guess_cn(const char *buf, int buflen)
+{
+    int i;
+    guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
+    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
+    guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
+    guess_dfa *top = NULL;
+
+    guess_dfa *order[] = { ORDER_CN, NULL };
+
+    for (i = 0; i < buflen; i++) {
+        int c = (unsigned char) buf[i];
+        int c2;
+
+        /* special treatment of iso-2022 escape sequence */
+        if (c == 0x1b) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                c2 = (unsigned char) buf[i + 2];
+                if (c == '$' && (c2 == ')' || c2 == '+'))
+                    return "ISO-2022-CN";
+            }
+        }
+
+        /* special treatment of BOM */
+        if (i == 0 && c == 0xff) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xfe)
+                    return UCS_2LE;
+            }
+        }
+        if (i == 0 && c == 0xfe) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xff)
+                    return UCS_2BE;
+            }
+        }
+
+        if (DFA_ALIVE(gb2312)) {
+            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030))
+                return "GB2312";
+            DFA_NEXT(gb2312, c);
+        }
+        if (DFA_ALIVE(utf8)) {
+            if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030))
+                return "UTF-8";
+            DFA_NEXT(utf8, c);
+        }
+        if (DFA_ALIVE(gb18030)) {
+            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312))
+                return "GB18030";
+            DFA_NEXT(gb18030, c);
+        }
+
+        if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
+            /* we ran out the possibilities */
+            return NULL;
+        }
+    }
+
+    /* Now, we have ambigous code.  Pick the highest score.  If more than
+       one candidate tie, pick the default encoding. */
+    for (i = 0; order[i] != NULL; i++) {
+        if (order[i]->state >= 0) { //DFA_ALIVE()
+            if (top == NULL || order[i]->score > top->score)
+                top = order[i];
+        }
+    }
+
+    if (top == &gb2312)
+        return "GB2312";
+    if (top == &utf8)
+        return "UTF-8";
+    if (top == &gb18030)
+        return "GB18030";
+    return NULL;
+}
+
+const char *guess_kr(const char *buf, int buflen)
+{
+    int i;
+    guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
+    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
+    guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
+    guess_dfa *top = NULL;
+
+    guess_dfa *order[] = { ORDER_KR, NULL };
+
+    for (i = 0; i < buflen; i++) {
+        int c = (unsigned char) buf[i];
+        int c2;
+
+        /* special treatment of iso-2022 escape sequence */
+        if (c == 0x1b) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                c2 = (unsigned char) buf[i + 2];
+                if (c == '$' && c2 == ')')
+                    return "ISO-2022-KR";
+            }
+        }
+
+        /* special treatment of BOM */
+        if (i == 0 && c == 0xff) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xfe)
+                    return UCS_2LE;
+            }
+        }
+        if (i == 0 && c == 0xfe) {
+            if (i < buflen - 1) {
+                c = (unsigned char) buf[i + 1];
+                if (c == 0xff)
+                    return UCS_2BE;
+            }
+        }
+
+        if (DFA_ALIVE(euck)) {
+            if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8))
+                return "EUC-KR";
+            DFA_NEXT(euck, c);
+        }
+        if (DFA_ALIVE(johab)) {
+            if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8))
+                return "JOHAB";
+            DFA_NEXT(johab, c);
+        }
+        if (DFA_ALIVE(utf8)) {
+            if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab))
+                return "UTF-8";
+            DFA_NEXT(utf8, c);
+        }
+
+        if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
+            /* we ran out the possibilities */
+            return NULL;
+        }
+    }
+
+    /* Now, we have ambigous code.  Pick the highest score.  If more than
+       one candidate tie, pick the default encoding. */
+    for (i = 0; order[i] != NULL; i++) {
+        if (order[i]->state >= 0) { //DFA_ALIVE()
+            if (top == NULL || order[i]->score > top->score)
+                top = order[i];
+        }
+    }
+
+    if (top == &euck)
+        return "EUC-KR";
+    if (top == &utf8)
+        return "UTF-8";
+    if (top == &johab)
+        return "JOHAB";
+    return NULL;
+}
--- a/src/libguess/guess.c	Wed Aug 01 08:13:05 2007 -0500
+++ b/src/libguess/guess.c	Thu Aug 02 01:38:15 2007 -0500
@@ -1,431 +1,4 @@
-/*
- * This code is derivative of guess.c of Gauche-0.8.3.
- * The following is the original copyright notice.
- */
-
-/*
- * guess.c - guessing character encoding 
- *
- *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
- * 
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- * 
- *   1. Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *
- *   2. Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in the
- *      documentation and/or other materials provided with the distribution.
- *
- *   3. Neither the name of the authors nor the names of its contributors
- *      may be used to endorse or promote products derived from this
- *      software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- *   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- *   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- *   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- *   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
 #include "libguess.h"
-#define NULL ((void *)0)
-
-/* take precedence if scores are same. you can customize the order as: */
-/* ORDER_** &highest, &second, ... &lowest */
-#define ORDER_JP &utf8, &sjis, &eucj
-#define ORDER_TW &utf8, &big5
-#define ORDER_CN &utf8, &gb2312, &gb18030
-#define ORDER_KR &utf8, &euck, &johab
-
-/* workaround for that glib's g_convert can't convert
-   properly from UCS-2BE/LE trailing after BOM. */
-#define WITH_G_CONVERT 1
-/* #undef WITH_G_CONVERT */
-
-#ifdef WITH_G_CONVERT
-const char UCS_2BE[] = "UTF-16";
-const char UCS_2LE[] = "UTF-16";
-#else
-const char UCS_2BE[] = "UCS-2BE";
-const char UCS_2LE[] = "UCS-2LE";
-#endif
-
-/* data types */
-typedef struct guess_arc_rec
-{
-    unsigned int next;          /* next state */
-    double score;               /* score */
-} guess_arc;
-
-typedef struct guess_dfa_rec
-{
-    signed char (*states)[256];
-    guess_arc *arcs;
-    int state;
-    double score;
-} guess_dfa;
-
-/* macros */
-#define DFA_INIT(st, ar) \
-    { st, ar, 0, 1.0 }
-
-#define DFA_NEXT(dfa, ch)                               \
-    do {                                                \
-        int arc__;                                      \
-        if (dfa.state >= 0) {                           \
-            arc__ = dfa.states[dfa.state][ch];          \
-            if (arc__ < 0) {                            \
-                dfa.state = -1;                         \
-            } else {                                    \
-                dfa.state = dfa.arcs[arc__].next;       \
-                dfa.score *= dfa.arcs[arc__].score;     \
-            }                                           \
-        }                                               \
-    } while (0)
-
-#define DFA_ALIVE(dfa)  (dfa.state >= 0)
-
-/* include DFA table generated by guess.scm */
-#include "guess_tab.c"
-
-
-int dfa_validate_utf8(const char *buf, int buflen)
-{
-    int i;
-    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
-
-    for (i = 0; i < buflen; i++) {
-        int c = (unsigned char) buf[i];
-
-        if (DFA_ALIVE(utf8))
-            DFA_NEXT(utf8, c);
-        else
-            break;
-    }
-
-    if(DFA_ALIVE(utf8))
-        return 1;
-    else 
-        return 0;
-}
-
-const char *guess_jp(const char *buf, int buflen)
-{
-    int i;
-    guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
-    guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
-    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
-    guess_dfa *top = NULL;
-
-    guess_dfa *order[] = { ORDER_JP, NULL };
-
-    for (i = 0; i < buflen; i++) {
-        int c = (unsigned char) buf[i];
-
-        /* special treatment of iso-2022 escape sequence */
-        if (c == 0x1b) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[++i];
-                if (c == '$' || c == '(')
-                    return "ISO-2022-JP";
-            }
-        }
-
-        /* special treatment of BOM */
-        if (i == 0 && c == 0xff) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xfe)
-                    return UCS_2LE;
-            }
-        }
-        if (i == 0 && c == 0xfe) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xff)
-                    return UCS_2BE;
-            }
-        }
-
-        if (DFA_ALIVE(eucj)) {
-            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8))
-                return "EUC-JP";
-            DFA_NEXT(eucj, c);
-        }
-        if (DFA_ALIVE(sjis)) {
-            if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8))
-                return "SJIS";
-            DFA_NEXT(sjis, c);
-        }
-        if (DFA_ALIVE(utf8)) {
-            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj))
-                return "UTF-8";
-            DFA_NEXT(utf8, c);
-        }
-
-        if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
-            /* we ran out the possibilities */
-            return NULL;
-        }
-    }
-
-    /* Now, we have ambigous code.  Pick the highest score.  If more than
-       one candidate tie, pick the default encoding. */
-    for (i = 0; order[i] != NULL; i++) {
-        if (order[i]->state >= 0) { //DFA_ALIVE()
-            if (top == NULL || order[i]->score > top->score)
-                top = order[i];
-        }
-    }
-
-    if (top == &eucj)
-        return "EUC-JP";
-    if (top == &utf8)
-        return "UTF-8";
-    if (top == &sjis)
-        return "SJIS";
-    return NULL;
-}
-
-const char *guess_tw(const char *buf, int buflen)
-{
-    int i;
-    guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
-    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
-    guess_dfa *top = NULL;
-
-    guess_dfa *order[] = { ORDER_TW, NULL };
-
-    for (i = 0; i < buflen; i++) {
-        int c = (unsigned char) buf[i];
-
-        /* special treatment of iso-2022 escape sequence */
-        if (c == 0x1b) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[++i];
-                if (c == '$' || c == '(')
-                    return "ISO-2022-TW";
-            }
-        }
-
-        /* special treatment of BOM */
-        if (i == 0 && c == 0xff) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xfe)
-                    return UCS_2LE;
-            }
-        }
-        if (i == 0 && c == 0xfe) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xff)
-                    return UCS_2BE;
-            }
-        }
-
-        if (DFA_ALIVE(big5)) {
-            if (!DFA_ALIVE(utf8))
-                return "BIG5";
-            DFA_NEXT(big5, c);
-        }
-        if (DFA_ALIVE(utf8)) {
-            if (!DFA_ALIVE(big5))
-                return "UTF-8";
-            DFA_NEXT(utf8, c);
-        }
-
-        if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
-            /* we ran out the possibilities */
-            return NULL;
-        }
-    }
-
-    /* Now, we have ambigous code.  Pick the highest score.  If more than
-       one candidate tie, pick the default encoding. */
-    for (i = 0; order[i] != NULL; i++) {
-        if (order[i]->state >= 0) { //DFA_ALIVE()
-            if (top == NULL || order[i]->score > top->score)
-                top = order[i];
-        }
-    }
-
-    if (top == &big5)
-        return "BIG5";
-    if (top == &utf8)
-        return "UTF-8";
-    return NULL;
-}
-
-const char *guess_cn(const char *buf, int buflen)
-{
-    int i;
-    guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
-    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
-    guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
-    guess_dfa *top = NULL;
-
-    guess_dfa *order[] = { ORDER_CN, NULL };
-
-    for (i = 0; i < buflen; i++) {
-        int c = (unsigned char) buf[i];
-        int c2;
-
-        /* special treatment of iso-2022 escape sequence */
-        if (c == 0x1b) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                c2 = (unsigned char) buf[i + 2];
-                if (c == '$' && (c2 == ')' || c2 == '+'))
-                    return "ISO-2022-CN";
-            }
-        }
-
-        /* special treatment of BOM */
-        if (i == 0 && c == 0xff) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xfe)
-                    return UCS_2LE;
-            }
-        }
-        if (i == 0 && c == 0xfe) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xff)
-                    return UCS_2BE;
-            }
-        }
-
-        if (DFA_ALIVE(gb2312)) {
-            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030))
-                return "GB2312";
-            DFA_NEXT(gb2312, c);
-        }
-        if (DFA_ALIVE(utf8)) {
-            if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030))
-                return "UTF-8";
-            DFA_NEXT(utf8, c);
-        }
-        if (DFA_ALIVE(gb18030)) {
-            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312))
-                return "GB18030";
-            DFA_NEXT(gb18030, c);
-        }
-
-        if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
-            /* we ran out the possibilities */
-            return NULL;
-        }
-    }
-
-    /* Now, we have ambigous code.  Pick the highest score.  If more than
-       one candidate tie, pick the default encoding. */
-    for (i = 0; order[i] != NULL; i++) {
-        if (order[i]->state >= 0) { //DFA_ALIVE()
-            if (top == NULL || order[i]->score > top->score)
-                top = order[i];
-        }
-    }
-
-    if (top == &gb2312)
-        return "GB2312";
-    if (top == &utf8)
-        return "UTF-8";
-    if (top == &gb18030)
-        return "GB18030";
-    return NULL;
-}
-
-const char *guess_kr(const char *buf, int buflen)
-{
-    int i;
-    guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
-    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
-    guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
-    guess_dfa *top = NULL;
-
-    guess_dfa *order[] = { ORDER_KR, NULL };
-
-    for (i = 0; i < buflen; i++) {
-        int c = (unsigned char) buf[i];
-        int c2;
-
-        /* special treatment of iso-2022 escape sequence */
-        if (c == 0x1b) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                c2 = (unsigned char) buf[i + 2];
-                if (c == '$' && c2 == ')')
-                    return "ISO-2022-KR";
-            }
-        }
-
-        /* special treatment of BOM */
-        if (i == 0 && c == 0xff) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xfe)
-                    return UCS_2LE;
-            }
-        }
-        if (i == 0 && c == 0xfe) {
-            if (i < buflen - 1) {
-                c = (unsigned char) buf[i + 1];
-                if (c == 0xff)
-                    return UCS_2BE;
-            }
-        }
-
-        if (DFA_ALIVE(euck)) {
-            if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8))
-                return "EUC-KR";
-            DFA_NEXT(euck, c);
-        }
-        if (DFA_ALIVE(johab)) {
-            if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8))
-                return "JOHAB";
-            DFA_NEXT(johab, c);
-        }
-        if (DFA_ALIVE(utf8)) {
-            if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab))
-                return "UTF-8";
-            DFA_NEXT(utf8, c);
-        }
-
-        if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
-            /* we ran out the possibilities */
-            return NULL;
-        }
-    }
-
-    /* Now, we have ambigous code.  Pick the highest score.  If more than
-       one candidate tie, pick the default encoding. */
-    for (i = 0; order[i] != NULL; i++) {
-        if (order[i]->state >= 0) { //DFA_ALIVE()
-            if (top == NULL || order[i]->score > top->score)
-                top = order[i];
-        }
-    }
-
-    if (top == &euck)
-        return "EUC-KR";
-    if (top == &utf8)
-        return "UTF-8";
-    if (top == &johab)
-        return "JOHAB";
-    return NULL;
-}
 
 typedef struct _guess_impl {
     struct _guess_impl *next;