annotate libguess/guess.c @ 1105:4be4d74db123 trunk

[svn] automatic character encoding detector for id3 metadata. --enable-chardet enables this feature.

/*
 * This code is derivative of guess.c of Gauche-0.8.3.
 * The following is the original copyright notice.
 */

/*
 * guess.c - guessing character encoding 
 *
 *   Copyright (c) 2000-2003 Shiro Kawai, All rights reserved.
 * 
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 * 
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 *   3. Neither the name of the authors nor the names of its contributors
 *      may be used to endorse or promote products derived from this
 *      software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 *   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 *   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 *   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 *   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 *   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "libguess.h"
#define NULL ((void *)0)

/* take precedence if scores are same. */
#undef PREFER_UTF8
#undef PREFER_SJIS
#undef PREFER_BIG5
#undef PREFER_GB18030
#undef PREFER_JOHAB

/* data types */
typedef struct guess_arc_rec {
    unsigned int next;          /* next state */
    double score;               /* score */
} guess_arc;

typedef struct guess_dfa_rec {
    signed char (*states)[256];
    guess_arc *arcs;
    int state;
    double score;
} guess_dfa;

/* macros */
#define DFA_INIT(st, ar) \
    { st, ar, 0, 1.0 }

#define DFA_NEXT(dfa, ch)                               \
    do {                                                \
        int arc__;                                      \
        if (dfa.state >= 0) {                           \
            arc__ = dfa.states[dfa.state][ch];          \
            if (arc__ < 0) {                            \
                dfa.state = -1;                         \
            } else {                                    \
                dfa.state = dfa.arcs[arc__].next;       \
                dfa.score *= dfa.arcs[arc__].score;     \
            }                                           \
        }                                               \
    } while (0)

#define DFA_ALIVE(dfa)  (dfa.state >= 0)

/* include DFA table generated by guess.scm */
#include "guess_tab.c"

const char *guess_jp(const char *buf, int buflen)
{
    int i;
    guess_dfa eucj = DFA_INIT(guess_eucj_st, guess_eucj_ar);
    guess_dfa sjis = DFA_INIT(guess_sjis_st, guess_sjis_ar);
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
    guess_dfa *top = NULL;

    for (i=0; i<buflen; i++) {
        int c = (unsigned char)buf[i];

        /* special treatment of jis escape sequence */
        if (c == 0x1b) {
            if (i < buflen-1) {
                c = (unsigned char)buf[++i];
                if (c == '$' || c == '(') return "ISO-2022-JP";
            }
        }
        
        /* special treatment of BOM */
        if (i==0 && c == 0xff) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xfe) return "UCS2-LE";
            }
        }
        if (i==0 && c == 0xfe) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xff) return "UCS2-BE";
            }
        }

        if (DFA_ALIVE(eucj)) {
            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) return "EUC-JP";
            DFA_NEXT(eucj, c);
        }
        if (DFA_ALIVE(sjis)) {
            if (!DFA_ALIVE(eucj) && !DFA_ALIVE(utf8)) return "SJIS";
            DFA_NEXT(sjis, c);
        }
        if (DFA_ALIVE(utf8)) {
            if (!DFA_ALIVE(sjis) && !DFA_ALIVE(eucj)) return "UTF-8";
            DFA_NEXT(utf8, c);
        }

        if (!DFA_ALIVE(eucj) && !DFA_ALIVE(sjis) && !DFA_ALIVE(utf8)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    /* Now, we have ambigous code.  Pick the highest score.  If more than
       one candidate tie, pick the default encoding. */
    if (DFA_ALIVE(eucj)) top = &eucj;
    if (DFA_ALIVE(utf8)) {
        if (top) {
#if defined PREFER_UTF8
            if (top->score <= utf8.score)  top = &utf8;
#else
            if (top->score <  utf8.score) top = &utf8;
#endif
        } else {
            top = &utf8;
        }
    }
    if (DFA_ALIVE(sjis)) {
        if (top) {
#if defined PREFER_SJIS
            if (top->score <= sjis.score)  top = &sjis;
#else
            if (top->score <  sjis.score) top = &sjis;
#endif
        } else {
            top = &sjis;
        }
    }

    if (top == &eucj) return "EUC-JP";
    if (top == &utf8) return "UTF-8";
    if (top == &sjis) return "SJIS";
    return NULL;
}

const char *guess_tw(const char *buf, int buflen)
{
    int i;
    guess_dfa big5 = DFA_INIT(guess_big5_st, guess_big5_ar);
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
    guess_dfa *top = NULL;

    for (i=0; i<buflen; i++) {
        int c = (unsigned char)buf[i];

        /* special treatment of jis escape sequence */
        if (c == 0x1b) {
            if (i < buflen-1) {
                c = (unsigned char)buf[++i];
                if (c == '$' || c == '(') return "ISO-2022-TW";
            }
        }
        
        /* special treatment of BOM */
        if (i==0 && c == 0xff) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xfe) return "UCS2-LE";
            }
        }
        if (i==0 && c == 0xfe) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xff) return "UCS2-BE";
            }
        }

        if (DFA_ALIVE(big5)) {
            if (!DFA_ALIVE(utf8)) return "BIG5";
            DFA_NEXT(big5, c);
        }
        if (DFA_ALIVE(utf8)) {
            if (!DFA_ALIVE(big5)) return "UTF-8";
            DFA_NEXT(utf8, c);
        }

        if (!DFA_ALIVE(big5) && !DFA_ALIVE(utf8)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    /* Now, we have ambigous code.  Pick the highest score.  If more than
       one candidate tie, pick the default encoding. */
    if (DFA_ALIVE(big5)) top = &big5;
    if (DFA_ALIVE(utf8)) {
        if (top) {
#if defined PREFER_UTF8
            if (top->score <= utf8.score)  top = &utf8;
#else
            if (top->score <  utf8.score) top = &utf8;
#endif
        } else {
            top = &utf8;
        }
    }

    if (top == &big5) return "BIG5";
    if (top == &utf8) return "UTF-8";
    return NULL;
}

const char *guess_cn(const char *buf, int buflen)
{
    int i;
    guess_dfa gb2312 = DFA_INIT(guess_gb2312_st, guess_gb2312_ar);
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
    guess_dfa gb18030 = DFA_INIT(guess_gb18030_st, guess_gb18030_ar);
    guess_dfa *top = NULL;

    for (i=0; i<buflen; i++) {
        int c = (unsigned char)buf[i];
        int c2;

        /* special treatment of jis escape sequence */
        if (c == 0x1b) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
		c2 = (unsigned char)buf[i+2];
                if (c == '$' && (c2 == ')' || c2 == '+')) return "ISO-2022-CN";
            }
        }
        
        if (DFA_ALIVE(gb2312)) {
            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) return "GB2312";
            DFA_NEXT(gb2312, c);
        }
        if (DFA_ALIVE(utf8)) {
            if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(gb18030)) return "UTF-8";
            DFA_NEXT(utf8, c);
        }
        if (DFA_ALIVE(gb18030)) {
            if (!DFA_ALIVE(utf8) && !DFA_ALIVE(gb2312)) return "GB18030";
            DFA_NEXT(gb18030, c);
        }

        if (!DFA_ALIVE(gb2312) && !DFA_ALIVE(utf8) && !DFA_ALIVE(gb18030)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    /* Now, we have ambigous code.  Pick the highest score.  If more than
       one candidate tie, pick the default encoding. */
    if (DFA_ALIVE(gb2312)) top = &gb2312;
    if (DFA_ALIVE(utf8)) {
        if (top) {
#if defined PREFER_UTF8
            if (top->score <= utf8.score)  top = &utf8;
#else
            if (top->score <  utf8.score) top = &utf8;
#endif
        } else {
            top = &utf8;
        }
    }
    if (DFA_ALIVE(gb18030)) {
        if (top) {
#if defined PREFER_GB18030
	        if (top->score <= gb18030.score) top = &gb18030;
#else
            if (top->score <  gb18030.score) top = &gb18030;
#endif
        } else {
            top = &gb18030;
        }
    }

    if (top == &gb2312) return "GB2312";
    if (top == &utf8)   return "UTF-8";
    if (top == &gb18030) return "GB18030";
    return NULL;
}

const char *guess_kr(const char *buf, int buflen)
{
    int i;
    guess_dfa euck = DFA_INIT(guess_euck_st, guess_euck_ar);
    guess_dfa utf8 = DFA_INIT(guess_utf8_st, guess_utf8_ar);
    guess_dfa johab = DFA_INIT(guess_johab_st, guess_johab_ar);
    guess_dfa *top = NULL;

    for (i=0; i<buflen; i++) {
        int c = (unsigned char)buf[i];
	int c2;

        /* special treatment of jis escape sequence */
        if (c == 0x1b) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
		c2 = (unsigned char)buf[i+2];
                if (c == '$' && c2 == ')') return "ISO-2022-KR";
            }
        }
        
        /* special treatment of BOM */
        if (i==0 && c == 0xff) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xfe) return "UCS2-LE";
            }
        }
        if (i==0 && c == 0xfe) {
            if (i < buflen-1) {
                c = (unsigned char)buf[i+1];
                if (c == 0xff) return "UCS2-BE";
            }
        }

        if (DFA_ALIVE(euck)) {
            if (!DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) return "EUC-KR";
            DFA_NEXT(euck, c);
        }
        if (DFA_ALIVE(johab)) {
            if (!DFA_ALIVE(euck) && !DFA_ALIVE(utf8)) return "JOHAB";
            DFA_NEXT(johab, c);
        }
        if (DFA_ALIVE(utf8)) {
            if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab)) return "UTF-8";
            DFA_NEXT(utf8, c);
        }

        if (!DFA_ALIVE(euck) && !DFA_ALIVE(johab) && !DFA_ALIVE(utf8)) {
            /* we ran out the possibilities */
            return NULL;
        }
    }

    /* Now, we have ambigous code.  Pick the highest score.  If more than
       one candidate tie, pick the default encoding. */
    if (DFA_ALIVE(euck)) top = &euck;
    if (DFA_ALIVE(utf8)) {
        if (top) {
#if defined PREFER_UTF8
            if (top->score <= utf8.score)  top = &utf8;
#else
            if (top->score <  utf8.score) top = &utf8;
#endif
        } else {
            top = &utf8;
        }
    }
    if (DFA_ALIVE(johab)) {
        if (top) {
#if defined PREFER_JOAHB
            if (top->score <= johab.score)  top = &johab;
#else
            if (top->score <  johab.score) top = &johab;
#endif
        } else {
            top = &johab;
        }
    }

    if (top == &euck)  return "EUC-KR";
    if (top == &utf8)  return "UTF-8";
    if (top == &johab) return "JOHAB";
    return NULL;
}