Mercurial > emacs
annotate lisp/international/utf-8.el @ 46601:64a36687ae30
(PATH_GAME): New macro, edited by ../Makefile.in.
author | Richard M. Stallman <rms@gnu.org> |
---|---|
date | Sun, 21 Jul 2002 20:29:33 +0000 |
parents | 395e5c46761b |
children | f0b8a25b0b7d |
rev | line source |
---|---|
46496 | 1 ;;; utf-8.el --- UTF-8 decoding/encoding support -*- coding: iso-2022-7bit -*- |
35542 | 2 |
3 ;; Copyright (C) 2001 Electrotechnical Laboratory, JAPAN. | |
4 ;; Licensed to the Free Software Foundation. | |
46496 | 5 ;; Copyright (C) 2001, 2002 Free Software Foundation, Inc. |
35542 | 6 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
7 ;; Author: TAKAHASHI Naoto <ntakahas@m17n.org> |
46496 | 8 ;; Maintainer: FSF |
36243 | 9 ;; Keywords: multilingual, Unicode, UTF-8, i18n |
35542 | 10 |
11 ;; This file is part of GNU Emacs. | |
12 | |
13 ;; GNU Emacs is free software; you can redistribute it and/or modify | |
14 ;; it under the terms of the GNU General Public License as published by | |
15 ;; the Free Software Foundation; either version 2, or (at your option) | |
16 ;; any later version. | |
17 | |
18 ;; GNU Emacs is distributed in the hope that it will be useful, | |
19 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 ;; GNU General Public License for more details. | |
22 | |
23 ;; You should have received a copy of the GNU General Public License | |
24 ;; along with GNU Emacs; see the file COPYING. If not, write to the | |
25 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
26 ;; Boston, MA 02111-1307, USA. | |
27 | |
28 ;;; Commentary: | |
29 | |
41873 | 30 ;; The coding-system `mule-utf-8' basically supports encoding/decoding |
31 ;; of the following character sets to and from UTF-8: | |
35542 | 32 ;; |
33 ;; ascii | |
34 ;; eight-bit-control | |
35 ;; latin-iso8859-1 | |
36 ;; mule-unicode-0100-24ff | |
37 ;; mule-unicode-2500-33ff | |
38 ;; mule-unicode-e000-ffff | |
39 ;; | |
36243 | 40 ;; On decoding, Unicode characters that do not fit into the above |
41 ;; character sets are handled as `eight-bit-control' or | |
42 ;; `eight-bit-graphic' characters to retain the information about the | |
46496 | 43 ;; original byte sequence and text properties record the corresponding |
44 ;; unicode. | |
45 ;; | |
46 ;; Fixme: note that reading and writing invalid utf-8 may not be | |
47 ;; idempotent -- to represent the bytes to fix that needs a new charset. | |
41873 | 48 ;; |
49 ;; Characters from other character sets can be encoded with | |
50 ;; mule-utf-8 by populating the table `ucs-mule-to-mule-unicode' and | |
46496 | 51 ;; registering the translation with `register-char-codings'. Hash |
52 ;; tables `utf-8-subst-table' and `utf-8-subst-rev-table' are used to | |
53 ;; support encoding and decoding of about a quarter of the CJK space | |
54 ;; between U+3400 and U+DFFF. | |
36243 | 55 |
56 ;; UTF-8 is defined in RFC 2279. A sketch of the encoding is: | |
35542 | 57 |
58 ;; scalar | utf-8 | |
59 ;; value | 1st byte | 2nd byte | 3rd byte | |
60 ;; --------------------+-----------+-----------+---------- | |
61 ;; 0000 0000 0xxx xxxx | 0xxx xxxx | | | |
62 ;; 0000 0yyy yyxx xxxx | 110y yyyy | 10xx xxxx | | |
63 ;; zzzz yyyy yyxx xxxx | 1110 zzzz | 10yy yyyy | 10xx xxxx | |
64 | |
65 ;;; Code: | |
66 | |
41873 | 67 (defvar ucs-mule-to-mule-unicode (make-translation-table) |
68 "Translation table for encoding to `mule-utf-8'.") | |
69 ;; Could have been done by ucs-tables loaded before. | |
70 (unless (get 'ucs-mule-to-mule-unicode 'translation-table) | |
46496 | 71 (define-translation-table 'ucs-mule-to-mule-unicode |
72 ucs-mule-to-mule-unicode)) | |
73 | |
74 (defvar utf-8-subst-table (make-hash-table :test 'eq)) | |
75 (defvar utf-8-subst-rev-table (make-hash-table :test 'eq)) | |
76 (define-translation-hash-table 'utf-8-subst-table utf-8-subst-table) | |
77 (define-translation-hash-table 'utf-8-subst-rev-table utf-8-subst-rev-table) | |
78 | |
79 (defvar utf-8-translation-table-for-decode (make-translation-table) | |
80 "Translation table applied after decoding utf-8 to mule-unicode. | |
81 This is only actually applied to characters which would normally be | |
82 decoded into mule-unicode-0100-24ff.") | |
83 (define-translation-table 'utf-8-translation-table-for-decode | |
84 utf-8-translation-table-for-decode) | |
85 | |
86 ;; Map Cyrillic and Greek to iso-8859 charsets, which take half the | |
87 ;; space of mule-unicode. For Latin scripts this isn't very | |
88 ;; important. Hebrew and Arabic might go here too when there's proper | |
89 ;; support for them. | |
90 (mapc | |
91 (lambda (pair) | |
92 (aset utf-8-translation-table-for-decode (car pair) (cdr pair))) | |
93 '((?$,1&d(B . ?,F4(B) (?$,1&e(B . ?,F5(B) (?$,1&f(B . ?,F6(B) (?$,1&h(B . ?,F8(B) (?$,1&i(B . ?,F9(B) | |
94 (?$,1&j(B . ?,F:(B) (?$,1&l(B . ?,F<(B) (?$,1&n(B . ?,F>(B) (?$,1&o(B . ?,F?(B) (?$,1&p(B . ?,F@(B) | |
95 (?$,1&q(B . ?,FA(B) (?$,1&r(B . ?,FB(B) (?$,1&s(B . ?,FC(B) (?$,1&t(B . ?,FD(B) (?$,1&u(B . ?,FE(B) | |
96 (?$,1&v(B . ?,FF(B) (?$,1&w(B . ?,FG(B) (?$,1&x(B . ?,FH(B) (?$,1&y(B . ?,FI(B) (?$,1&z(B . ?,FJ(B) | |
97 (?$,1&{(B . ?,FK(B) (?$,1&|(B . ?,FL(B) (?$,1&}(B . ?,FM(B) (?$,1&~(B . ?,FN(B) (?$,1&(B . ?,FO(B) | |
98 (?$,1' (B . ?,FP(B) (?$,1'!(B . ?,FQ(B) (?$,1'#(B . ?,FS(B) (?$,1'$(B . ?,FT(B) (?$,1'%(B . ?,FU(B) | |
99 (?$,1'&(B . ?,FV(B) (?$,1''(B . ?,FW(B) (?$,1'((B . ?,FX(B) (?$,1')(B . ?,FY(B) (?$,1'*(B . ?,FZ(B) | |
100 (?$,1'+(B . ?,F[(B) (?$,1',(B . ?,F\(B) (?$,1'-(B . ?,F](B) (?$,1'.(B . ?,F^(B) (?$,1'/(B . ?,F_(B) | |
101 (?$,1'0(B . ?,F`(B) (?$,1'1(B . ?,Fa(B) (?$,1'2(B . ?,Fb(B) (?$,1'3(B . ?,Fc(B) (?$,1'4(B . ?,Fd(B) | |
102 (?$,1'5(B . ?,Fe(B) (?$,1'6(B . ?,Ff(B) (?$,1'7(B . ?,Fg(B) (?$,1'8(B . ?,Fh(B) (?$,1'9(B . ?,Fi(B) | |
103 (?$,1':(B . ?,Fj(B) (?$,1';(B . ?,Fk(B) (?$,1'<(B . ?,Fl(B) (?$,1'=(B . ?,Fm(B) (?$,1'>(B . ?,Fn(B) | |
104 (?$,1'?(B . ?,Fo(B) (?$,1'@(B . ?,Fp(B) (?$,1'A(B . ?,Fq(B) (?$,1'B(B . ?,Fr(B) (?$,1'C(B . ?,Fs(B) | |
105 (?$,1'D(B . ?,Ft(B) (?$,1'E(B . ?,Fu(B) (?$,1'F(B . ?,Fv(B) (?$,1'G(B . ?,Fw(B) (?$,1'H(B . ?,Fx(B) | |
106 (?$,1'I(B . ?,Fy(B) (?$,1'J(B . ?,Fz(B) (?$,1'K(B . ?,F{(B) (?$,1'L(B . ?,F|(B) (?$,1'M(B . ?,F}(B) | |
107 (?$,1'N(B . ?,F~(B) | |
108 | |
109 (?$,1(!(B . ?,L!(B) (?$,1("(B . ?,L"(B) (?$,1(#(B . ?,L#(B) (?$,1($(B . ?,L$(B) | |
110 (?$,1(%(B . ?,L%(B) (?$,1(&(B . ?,L&(B) (?$,1('(B . ?,L'(B) (?$,1(((B . ?,L((B) (?$,1()(B . ?,L)(B) | |
111 (?$,1(*(B . ?,L*(B) (?$,1(+(B . ?,L+(B) (?$,1(,(B . ?,L,(B) (?$,1(.(B . ?,L.(B) (?$,1(/(B . ?,L/(B) | |
112 (?$,1(0(B . ?,L0(B) (?$,1(1(B . ?,L1(B) (?$,1(2(B . ?,L2(B) (?$,1(3(B . ?,L3(B) (?$,1(4(B . ?,L4(B) | |
113 (?$,1(5(B . ?,L5(B) (?$,1(6(B . ?,L6(B) (?$,1(7(B . ?,L7(B) (?$,1(8(B . ?,L8(B) (?$,1(9(B . ?,L9(B) | |
114 (?$,1(:(B . ?,L:(B) (?$,1(;(B . ?,L;(B) (?$,1(<(B . ?,L<(B) (?$,1(=(B . ?,L=(B) (?$,1(>(B . ?,L>(B) | |
115 (?$,1(?(B . ?,L?(B) (?$,1(@(B . ?,L@(B) (?$,1(A(B . ?,LA(B) (?$,1(B(B . ?,LB(B) (?$,1(C(B . ?,LC(B) | |
116 (?$,1(D(B . ?,LD(B) (?$,1(E(B . ?,LE(B) (?$,1(F(B . ?,LF(B) (?$,1(G(B . ?,LG(B) (?$,1(H(B . ?,LH(B) | |
117 (?$,1(I(B . ?,LI(B) (?$,1(J(B . ?,LJ(B) (?$,1(K(B . ?,LK(B) (?$,1(L(B . ?,LL(B) (?$,1(M(B . ?,LM(B) | |
118 (?$,1(N(B . ?,LN(B) (?$,1(O(B . ?,LO(B) (?$,1(P(B . ?,LP(B) (?$,1(Q(B . ?,LQ(B) (?$,1(R(B . ?,LR(B) | |
119 (?$,1(S(B . ?,LS(B) (?$,1(T(B . ?,LT(B) (?$,1(U(B . ?,LU(B) (?$,1(V(B . ?,LV(B) (?$,1(W(B . ?,LW(B) | |
120 (?$,1(X(B . ?,LX(B) (?$,1(Y(B . ?,LY(B) (?$,1(Z(B . ?,LZ(B) (?$,1([(B . ?,L[(B) (?$,1(\(B . ?,L\(B) | |
121 (?$,1(](B . ?,L](B) (?$,1(^(B . ?,L^(B) (?$,1(_(B . ?,L_(B) (?$,1(`(B . ?,L`(B) (?$,1(a(B . ?,La(B) | |
122 (?$,1(b(B . ?,Lb(B) (?$,1(c(B . ?,Lc(B) (?$,1(d(B . ?,Ld(B) (?$,1(e(B . ?,Le(B) (?$,1(f(B . ?,Lf(B) | |
123 (?$,1(g(B . ?,Lg(B) (?$,1(h(B . ?,Lh(B) (?$,1(i(B . ?,Li(B) (?$,1(j(B . ?,Lj(B) (?$,1(k(B . ?,Lk(B) | |
124 (?$,1(l(B . ?,Ll(B) (?$,1(m(B . ?,Lm(B) (?$,1(n(B . ?,Ln(B) (?$,1(o(B . ?,Lo(B) (?$,1(q(B . ?,Lq(B) | |
125 (?$,1(r(B . ?,Lr(B) (?$,1(s(B . ?,Ls(B) (?$,1(t(B . ?,Lt(B) (?$,1(u(B . ?,Lu(B) (?$,1(v(B . ?,Lv(B) | |
126 (?$,1(w(B . ?,Lw(B) (?$,1(x(B . ?,Lx(B) (?$,1(y(B . ?,Ly(B) (?$,1(z(B . ?,Lz(B) (?$,1({(B . ?,L{(B) | |
127 (?$,1(|(B . ?,L|(B) (?$,1(~(B . ?,L~(B) (?$,1((B . ?,L(B))) | |
128 | |
129 (defcustom utf-8-fragment-on-decoding nil | |
130 "Whether or not to decode some scripts in UTF-8 text into 8-bit characters. | |
131 Setting this means that the relevant Cyrillic and Greek characters are | |
132 decoded into the iso8859 charsets rather than into | |
133 mule-unicode-0100-24ff. The 8-bit characters take half as much space | |
134 in the buffer, but using them may affect how the buffer can be re-encoded | |
135 and may require a different input method to search for them, for instance. | |
136 See `unify-8859-on-decoding-mode' and `unify-8859-on-encoding-mode' | |
137 for mechanisms to make this largely transparent." | |
138 :set (lambda (s v) | |
139 (if v | |
140 (define-translation-table 'utf-8-translation-table-for-decode | |
141 utf-8-translation-table-for-decode) | |
142 (define-translation-table 'utf-8-translation-table-for-decode)) | |
143 (set-default s v)) | |
144 :version "21.4" | |
145 :type 'boolean | |
146 :group 'mule) | |
147 | |
148 (defcustom utf-8-translate-cjk nil | |
149 "Whether the `mule-utf-8' coding system should encode many CJK characters. | |
150 | |
151 Enabling this loads tables which enable the coding system to encode | |
152 characters in the charsets `korean-ksc5601', `chinese-gb2312' and | |
153 `japanese-jisx0208', and to decode the corresponding unicodes into | |
154 such characters. This works by loading the library `utf-8-subst'; see | |
155 its commentary. The tables are fairly large (about 33000 entries), so this | |
156 option is not the default." | |
157 :link '(emacs-commentary-link "utf-8-subst") | |
158 :set (lambda (s v) | |
159 (when v | |
160 (require 'utf-8-subst) | |
161 (let ((table (make-char-table 'translation-table))) | |
162 (coding-system-put 'mule-utf-8 'safe-charsets | |
163 (append (coding-system-get 'mule-utf-8 | |
164 'safe-charsets) | |
165 '(korean-ksc5601 chinese-gb2312 | |
166 japanese-jisx0208))) | |
167 (maphash (lambda (k v) | |
168 (aset table k v)) | |
169 utf-8-subst-rev-table) | |
170 (register-char-codings 'mule-utf-8 table))) | |
171 (set-default s v)) | |
172 :version "21.4" | |
173 :type 'boolean | |
174 :group 'mule) | |
175 | |
35542 | 176 (define-ccl-program ccl-decode-mule-utf-8 |
177 ;; | |
178 ;; charset | bytes in utf-8 | bytes in emacs | |
179 ;; -----------------------+----------------+--------------- | |
180 ;; ascii | 1 | 1 | |
181 ;; -----------------------+----------------+--------------- | |
182 ;; eight-bit-control | 2 | 2 | |
41873 | 183 ;; eight-bit-graphic | 2 | 1 |
35542 | 184 ;; latin-iso8859-1 | 2 | 2 |
185 ;; -----------------------+----------------+--------------- | |
186 ;; mule-unicode-0100-24ff | 2 | 4 | |
187 ;; (< 0800) | | | |
188 ;; -----------------------+----------------+--------------- | |
189 ;; mule-unicode-0100-24ff | 3 | 4 | |
190 ;; (>= 8000) | | | |
191 ;; mule-unicode-2500-33ff | 3 | 4 | |
192 ;; mule-unicode-e000-ffff | 3 | 4 | |
193 ;; | |
194 ;; Thus magnification factor is two. | |
195 ;; | |
196 `(2 | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
197 ((r5 = ,(charset-id 'eight-bit-control)) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
198 (r6 = ,(charset-id 'eight-bit-graphic)) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
199 (loop |
35542 | 200 (read r0) |
201 | |
202 ;; 1byte encoding, i.e., ascii | |
203 (if (r0 < #x80) | |
204 (write r0) | |
46496 | 205 (if (r0 < #xc0) ; continuation byte (invalid here) |
206 (if (r0 < #xa0) | |
207 (write-multibyte-character r5 r0) | |
208 (write-multibyte-character r6 r0)) | |
209 ;; 2 byte encoding 00000yyyyyxxxxxx = 110yyyyy 10xxxxxx | |
210 (if (r0 < #xe0) | |
211 ((read r1) | |
35542 | 212 |
46496 | 213 (if ((r1 & #b11000000) != #b10000000) |
214 ;; Invalid 2-byte sequence | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
215 ((if (r0 < #xa0) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
216 (write-multibyte-character r5 r0) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
217 (write-multibyte-character r6 r0)) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
218 (if (r1 < #x80) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
219 (write r1) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
220 (if (r1 < #xa0) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
221 (write-multibyte-character r5 r1) |
46496 | 222 (write-multibyte-character r6 r1)))) |
223 | |
224 ((r3 = r0) ; save in case of overlong sequence | |
225 (r2 = r1) | |
226 (r0 &= #x1f) | |
227 (r0 <<= 6) | |
228 (r2 = r1) ; save in case of overlong sequence | |
229 (r1 &= #x3f) | |
230 (r1 += r0) | |
231 ;; Now r1 holds scalar value | |
232 | |
233 (if (r1 < 128) ; `overlong sequence' | |
234 ((if (r3 < #xa0) | |
235 (write-multibyte-character r5 r3) | |
236 (write-multibyte-character r6 r3)) | |
237 (if (r2 < #x80) | |
238 (write r2) | |
239 (if (r2 < #xa0) | |
240 (write-multibyte-character r5 r2) | |
241 (write-multibyte-character r6 r2)))) | |
242 | |
243 ;; eight-bit-control | |
244 (if (r1 < 160) | |
245 ((write-multibyte-character r5 r1)) | |
246 | |
247 ;; latin-iso8859-1 | |
248 (if (r1 < 256) | |
249 ((r0 = ,(charset-id 'latin-iso8859-1)) | |
250 (r1 -= 128) | |
251 (write-multibyte-character r0 r1)) | |
252 | |
253 ;; mule-unicode-0100-24ff (< 0800) | |
254 ((r0 = ,(charset-id 'mule-unicode-0100-24ff)) | |
255 (r1 -= #x0100) | |
256 (r2 = (((r1 / 96) + 32) << 7)) | |
257 (r1 %= 96) | |
258 (r1 += (r2 + 32)) | |
259 (translate-character | |
260 utf-8-translation-table-for-decode r0 r1) | |
261 (write-multibyte-character r0 r1)))))))) | |
262 | |
263 ;; 3byte encoding | |
264 ;; zzzzyyyyyyxxxxxx = 1110zzzz 10yyyyyy 10xxxxxx | |
265 (if (r0 < #xf0) | |
266 ((read r1 r2) | |
267 | |
268 ;; This is set to 1 if the encoding is invalid. | |
269 (r4 = 0) | |
270 | |
271 (r3 = (r1 & #b11000000)) | |
272 (r3 |= ((r2 >> 2) & #b00110000)) | |
273 (if (r3 != #b10100000) | |
274 (r4 = 1) | |
275 ((r3 = ((r0 & #x0f) << 12)) | |
276 (r3 += ((r1 & #x3f) << 6)) | |
277 (r3 += (r2 & #x3f)) | |
278 (if (r3 < #x0800) | |
279 (r4 = 1)))) | |
280 | |
281 (if (r4 != 0) | |
282 ;; Invalid 3-byte sequence | |
283 ((if (r0 < #xa0) | |
284 (write-multibyte-character r5 r0) | |
285 (write-multibyte-character r6 r0)) | |
286 (if (r1 < #x80) | |
287 (write r1) | |
288 (if (r1 < #xa0) | |
289 (write-multibyte-character r5 r1) | |
290 (write-multibyte-character r6 r1))) | |
291 (if (r2 < #x80) | |
292 (write r2) | |
293 (if (r2 < #xa0) | |
294 (write-multibyte-character r5 r2) | |
295 (write-multibyte-character r6 r2)))) | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
296 |
46496 | 297 ;; mule-unicode-0100-24ff (>= 0800) |
298 ((if (r3 < #x2500) | |
299 ((r0 = ,(charset-id 'mule-unicode-0100-24ff)) | |
300 (r3 -= #x0100) | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
301 (r3 //= 96) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
302 (r1 = (r7 + 32)) |
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
303 (r1 += ((r3 + 32) << 7)) |
46496 | 304 (translate-character |
305 utf-8-translation-table-for-decode r0 r1) | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
306 (write-multibyte-character r0 r1)) |
46496 | 307 |
308 ;; mule-unicode-2500-33ff | |
309 ;; Fixme: Perhaps allow translation via | |
310 ;; utf-8-subst-table for #x2e80 up, so that we use | |
311 ;; consistent charsets for all of CJK. Would need | |
312 ;; corresponding change to encoding tables. | |
313 (if (r3 < #x3400) | |
314 ((r0 = ,(charset-id 'mule-unicode-2500-33ff)) | |
315 (r3 -= #x2500) | |
316 (r3 //= 96) | |
317 (r1 = (r7 + 32)) | |
318 (r1 += ((r3 + 32) << 7)) | |
319 (write-multibyte-character r0 r1)) | |
320 | |
321 ;; U+3400 .. U+D7FF | |
322 ;; Try to convert to CJK chars, else keep | |
323 ;; them as eight-bit-{control|graphic}. | |
324 (if (r3 < #xd800) | |
325 ((r4 = r3) ; don't zap r3 | |
326 (lookup-integer utf-8-subst-table r4 r5) | |
327 (if r7 | |
328 ;; got a translation | |
329 ((write-multibyte-character r4 r5) | |
330 ;; Zapped through register starvation. | |
331 (r5 = ,(charset-id 'eight-bit-control))) | |
332 ;; #xe0 <= r0 < #xf0, so r0 is eight-bit-graphic | |
333 ((r3 = r6) | |
334 (write-multibyte-character r3 r0) | |
335 (if (r1 < #xa0) | |
336 (r3 = r5)) | |
337 (write-multibyte-character r3 r1) | |
338 (if (r2 < #xa0) | |
339 (r3 = r5) | |
340 (r3 = r6)) | |
341 (write-multibyte-character r3 r2)))) | |
342 | |
343 ;; Surrogates, U+D800 .. U+DFFF | |
344 ;; Fixme: process them properly. | |
345 (if (r3 < #xe000) | |
346 ((r3 = r6) | |
347 (write-multibyte-character r3 r0) ; eight-bit-graphic | |
348 (if (r1 < #xa0) | |
349 (r3 = r5)) | |
350 (write-multibyte-character r3 r1) | |
351 (if (r2 < #xa0) | |
352 (r3 = r5) | |
353 (r3 = r6)) | |
354 (write-multibyte-character r3 r2)) | |
37934
88389fa9b713
(ccl-decode-mule-utf-8): Handle
Gerd Moellmann <gerd@gnu.org>
parents:
37097
diff
changeset
|
355 |
46496 | 356 ;; mule-unicode-e000-ffff |
357 ;; Fixme: fffe and ffff are invalid. | |
358 ((r0 = ,(charset-id 'mule-unicode-e000-ffff)) | |
359 (r3 -= #xe000) | |
360 (r3 //= 96) | |
361 (r1 = (r7 + 32)) | |
362 (r1 += ((r3 + 32) << 7)) | |
363 (write-multibyte-character r0 r1))))))))) | |
35542 | 364 |
46496 | 365 (if (r0 < #xfe) |
366 ;; 4byte encoding | |
367 ;; keep those bytes as eight-bit-{control|graphic} | |
368 ;; Fixme: allow lookup in utf-8-subst-table. | |
369 ((read r1 r2 r3) | |
370 ;; r0 > #xf0, thus eight-bit-graphic | |
371 (write-multibyte-character r6 r0) | |
372 (if (r1 < #xa0) | |
373 (if (r1 < #x80) ; invalid byte | |
374 (write r1) | |
375 (write-multibyte-character r5 r1)) | |
376 (write-multibyte-character r6 r1)) | |
377 (if (r2 < #xa0) | |
378 (if (r2 < #x80) ; invalid byte | |
379 (write r2) | |
380 (write-multibyte-character r5 r2)) | |
381 (write-multibyte-character r6 r2)) | |
382 (if (r3 < #xa0) | |
383 (if (r3 < #x80) ; invalid byte | |
384 (write r3) | |
385 (write-multibyte-character r5 r3)) | |
386 (write-multibyte-character r6 r3)) | |
387 (if (r0 >= #xf8) ; 5- or 6-byte encoding | |
388 ((read r1) | |
389 (if (r1 < #xa0) | |
390 (if (r1 < #x80) ; invalid byte | |
391 (write r1) | |
392 (write-multibyte-character r5 r1)) | |
393 (write-multibyte-character r6 r1)) | |
394 (if (r0 >= #xfc) ; 6-byte | |
395 ((read r1) | |
396 (if (r1 < #xa0) | |
397 (if (r1 < #x80) ; invalid byte | |
398 (write r1) | |
399 (write-multibyte-character r5 r1)) | |
400 (write-multibyte-character r6 r1))))))) | |
401 ;; else invalid byte >= #xfe | |
402 (write-multibyte-character r6 r0)))))) | |
35542 | 403 (repeat)))) |
404 | |
36243 | 405 "CCL program to decode UTF-8. |
36465 | 406 Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
46496 | 407 mule-unicode-*, but see also `utf-8-translation-table-for-decode' and |
408 `utf-8-subst-table'. | |
409 Encodings of un-representable Unicode characters are decoded asis into | |
410 eight-bit-control and eight-bit-graphic characters.") | |
35542 | 411 |
412 (define-ccl-program ccl-encode-mule-utf-8 | |
413 `(1 | |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
414 ((r5 = -1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
415 (loop |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
416 (if (r5 < 0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
417 ((r1 = -1) |
41873 | 418 (read-multibyte-character r0 r1) |
419 (translate-character ucs-mule-to-mule-unicode r0 r1)) | |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
420 (;; We have already done read-multibyte-character. |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
421 (r0 = r5) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
422 (r1 = r6) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
423 (r5 = -1))) |
35542 | 424 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
425 (if (r0 == ,(charset-id 'ascii)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
426 (write r1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
427 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
428 (if (r0 == ,(charset-id 'latin-iso8859-1)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
429 ;; r1 scalar utf-8 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
430 ;; 0000 0yyy yyxx xxxx 110y yyyy 10xx xxxx |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
431 ;; 20 0000 0000 1010 0000 1100 0010 1010 0000 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
432 ;; 7f 0000 0000 1111 1111 1100 0011 1011 1111 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
433 ((r0 = (((r1 & #x40) >> 6) | #xc2)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
434 (r1 &= #x3f) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
435 (r1 |= #x80) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
436 (write r0 r1)) |
35542 | 437 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
438 (if (r0 == ,(charset-id 'mule-unicode-0100-24ff)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
439 ((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
440 ;; #x3f80 == (0011 1111 1000 0000)b |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
441 (r1 &= #x7f) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
442 (r1 += (r0 + 224)) ; 240 == -32 + #x0100 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
443 ;; now r1 holds scalar value |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
444 (if (r1 < #x0800) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
445 ;; 2byte encoding |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
446 ((r0 = (((r1 & #x07c0) >> 6) | #xc0)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
447 ;; #x07c0 == (0000 0111 1100 0000)b |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
448 (r1 &= #x3f) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
449 (r1 |= #x80) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
450 (write r0 r1)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
451 ;; 3byte encoding |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
452 ((r0 = (((r1 & #xf000) >> 12) | #xe0)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
453 (r2 = ((r1 & #x3f) | #x80)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
454 (r1 &= #x0fc0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
455 (r1 >>= 6) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
456 (r1 |= #x80) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
457 (write r0 r1 r2)))) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
458 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
459 (if (r0 == ,(charset-id 'mule-unicode-2500-33ff)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
460 ((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
461 (r1 &= #x7f) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
462 (r1 += (r0 + 9440)) ; 9440 == -32 + #x2500 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
463 (r0 = (((r1 & #xf000) >> 12) | #xe0)) |
35542 | 464 (r2 = ((r1 & #x3f) | #x80)) |
465 (r1 &= #x0fc0) | |
466 (r1 >>= 6) | |
467 (r1 |= #x80) | |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
468 (write r0 r1 r2)) |
35542 | 469 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
470 (if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
471 ((r0 = ((((r1 & #x3f80) >> 7) - 32) * 96)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
472 (r1 &= #x7f) |
46496 | 473 (r1 += (r0 + 57312)) ; 57312 == -32 + #xe000 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
474 (r0 = (((r1 & #xf000) >> 12) | #xe0)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
475 (r2 = ((r1 & #x3f) | #x80)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
476 (r1 &= #x0fc0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
477 (r1 >>= 6) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
478 (r1 |= #x80) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
479 (write r0 r1 r2)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
480 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
481 (if (r0 == ,(charset-id 'eight-bit-control)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
482 ;; r1 scalar utf-8 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
483 ;; 0000 0yyy yyxx xxxx 110y yyyy 10xx xxxx |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
484 ;; 80 0000 0000 1000 0000 1100 0010 1000 0000 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
485 ;; 9f 0000 0000 1001 1111 1100 0010 1001 1111 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
486 ((write #xc2) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
487 (write r1)) |
35542 | 488 |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
489 (if (r0 == ,(charset-id 'eight-bit-graphic)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
490 ;; r1 scalar utf-8 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
491 ;; 0000 0yyy yyxx xxxx 110y yyyy 10xx xxxx |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
492 ;; a0 0000 0000 1010 0000 1100 0010 1010 0000 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
493 ;; ff 0000 0000 1111 1111 1101 1111 1011 1111 |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
494 ((write r1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
495 (r1 = -1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
496 (read-multibyte-character r0 r1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
497 (if (r0 != ,(charset-id 'eight-bit-graphic)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
498 (if (r0 != ,(charset-id 'eight-bit-control)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
499 ((r5 = r0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
500 (r6 = r1)))) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
501 (if (r5 < 0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
502 ((read-multibyte-character r0 r2) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
503 (if (r0 != ,(charset-id 'eight-bit-graphic)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
504 (if (r0 != ,(charset-id 'eight-bit-control)) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
505 ((r5 = r0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
506 (r6 = r2)))) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
507 (if (r5 < 0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
508 (write r1 r2) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
509 (if (r1 < #xa0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
510 (write r1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
511 ((write #xc2) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
512 (write r1))))))) |
35542 | 513 |
46496 | 514 ((lookup-character utf-8-subst-rev-table r0 r1) |
515 (if r7 ; lookup succeeded | |
516 ((r1 = (((r0 & #xf000) >> 12) | #xe0)) | |
517 (r2 = ((r0 & #x3f) | #x80)) | |
518 (r0 &= #x0fc0) | |
519 (r0 >>= 6) | |
520 (r0 |= #x80) | |
521 (write r1 r0 r2)) | |
522 ;; Unsupported character. | |
523 ;; Output U+FFFD, which is `ef bf bd' in UTF-8. | |
524 ((write #xef) | |
525 (write #xbf) | |
526 (write #xbd))))))))))) | |
37097
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
527 (repeat))) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
528 (if (r1 >= #xa0) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
529 (write r1) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
530 (if (r1 >= #x80) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
531 ((write #xc2) |
b095952a8678
(ccl-encode-mule-utf-8): Fix handling of eight-bit-control chars.
Kenichi Handa <handa@m17n.org>
parents:
36522
diff
changeset
|
532 (write r1))))) |
35542 | 533 |
46496 | 534 "CCL program to encode into UTF-8.") |
35542 | 535 |
41873 | 536 ;; Dummy definition so that the CCL can be checked correctly; the |
537 ;; actual data are loaded on demand. | |
538 (unless (boundp 'ucs-mule-8859-to-mule-unicode) ; don't zap it | |
539 (define-translation-table 'ucs-mule-8859-to-mule-unicode)) | |
540 | |
46496 | 541 (define-ccl-program ccl-untranslated-to-ucs |
542 `(0 | |
543 (if (r0 < #xf0) ; 3-byte encoding, as above | |
544 ((r4 = 0) | |
545 (r3 = (r1 & #b11000000)) | |
546 (r3 |= ((r2 >> 2) & #b00110000)) | |
547 (if (r3 != #b10100000) | |
548 (r4 = 1) | |
549 ((r3 = ((r0 & #x0f) << 12)) | |
550 (r3 += ((r1 & #x3f) << 6)) | |
551 (r3 += (r2 & #x3f)) | |
552 (if (r3 < #x0800) | |
553 (r4 = 1)))) | |
554 (if (r4 != 0) | |
555 (r0 = 0) | |
556 (r0 = r3))) | |
557 (if (r0 < #xf8) ; 4-byte (Mule-UCS recipe) | |
558 ((r4 = (r1 >> 6)) | |
559 (if (r4 != #b10) | |
560 (r0 = 0) | |
561 ((r4 = (r2 >> 6)) | |
562 (if (r4 != #b10) | |
563 (r0 = 0) | |
564 ((r4 = (r3 >> 6)) | |
565 (if (r4 != #b10) | |
566 (r0 = 0) | |
567 ((r1 = ((r1 & #x3F) << 12)) | |
568 (r2 = ((r2 & #x3F) << 6)) | |
569 (r3 &= #x3F) | |
570 (r0 = (((((r0 & #x07) << 18) | r1) | r2) | r3))))))))) | |
571 (r0 = 0)))) | |
572 "Decode 3- or 4-byte sequences in r0, r1, r2 [,r3] to unicodes in r0. | |
573 r0 == 0 for invalid sequence.") | |
574 | |
575 (defvar utf-8-ccl-regs (make-vector 8 0)) | |
576 | |
41873 | 577 (defsubst utf-8-untranslated-to-ucs () |
46496 | 578 "Return the UCS code for an untranslated sequence of raw bytes t point. |
579 Only for 3- or 4-byte sequences." | |
580 (aset utf-8-ccl-regs 0 (or (char-after) 0)) | |
581 (aset utf-8-ccl-regs 1 (or (char-after (1+ (point))) 0)) | |
582 (aset utf-8-ccl-regs 2 (or (char-after (+ 2 (point))) 0)) | |
583 (aset utf-8-ccl-regs 3 (or (char-after (+ 3 (point))) 0)) | |
584 (ccl-execute 'ccl-untranslated-to-ucs utf-8-ccl-regs) | |
585 (aref utf-8-ccl-regs 0)) | |
41873 | 586 |
587 (defun utf-8-help-echo (window object position) | |
588 (format "Untranslated Unicode U+%04X" | |
589 (get-char-property position 'untranslated-utf-8 object))) | |
590 | |
591 ;; We compose the untranslatable sequences into a single character. | |
592 ;; This is infelicitous for editing, because there's currently no | |
593 ;; mechanism for treating compositions as atomic, but is OK for | |
46496 | 594 ;; display. They are composed to U+FFFD with help-echo which |
595 ;; indicates the unicodes they represent. This function GCs too much. | |
41873 | 596 (defsubst utf-8-compose () |
597 "Put a suitable composition on an untranslatable sequence. | |
598 Return the sequence's length." | |
599 (let* ((u (utf-8-untranslated-to-ucs)) | |
46496 | 600 (l (unless (zerop u) |
601 (if (>= u #x10000) | |
41873 | 602 4 |
46496 | 603 3)))) |
604 (when l | |
41873 | 605 (put-text-property (point) (min (point-max) (+ l (point))) |
606 'untranslated-utf-8 u) | |
46496 | 607 (put-text-property (point) (min (point-max) (+ l (point))) |
608 'help-echo 'utf-8-help-echo) | |
609 (compose-region (point) (+ l (point)) ?$,3u=(B) | |
41873 | 610 l))) |
611 | |
612 (defcustom utf-8-compose-scripts nil | |
46496 | 613 "*Non-nil means compose various scripts on decoding utf-8 text." |
41873 | 614 :group 'mule |
46496 | 615 :version "21.4" |
616 :type 'boolean) | |
41873 | 617 |
618 (defun utf-8-post-read-conversion (length) | |
619 "Compose untranslated utf-8 sequences into single characters. | |
620 Also compose particular scripts if `utf-8-compose-scripts' is non-nil." | |
621 (save-excursion | |
622 ;; Can't do eval-when-compile to insert a multibyte constant | |
623 ;; version of the string in the loop, since it's always loaded as | |
624 ;; unibyte from a byte-compiled file. | |
46496 | 625 (let ((range (string-as-multibyte "^\xe1-\xf7"))) |
626 (while (and (skip-chars-forward range) | |
41873 | 627 (not (eobp))) |
628 (forward-char (utf-8-compose))))) | |
46496 | 629 ;; Fixme: Takahashi-san implies it may not work this easily. I |
630 ;; asked why but didn't get a reply. -- fx | |
41873 | 631 (when (and utf-8-compose-scripts (> length 1)) |
632 ;; These currently have definitions which cover the relevant | |
46496 | 633 ;; unicodes. We could avoid loading thai-util &c by checking |
41873 | 634 ;; whether the region contains any characters with the appropriate |
635 ;; categories. There aren't yet Unicode-based rules for Tibetan. | |
636 (save-excursion (setq length (diacritic-post-read-conversion length))) | |
637 (save-excursion (setq length (thai-post-read-conversion length))) | |
638 (save-excursion (setq length (lao-post-read-conversion length))) | |
46496 | 639 (save-excursion |
640 (setq length (in-is13194-devanagari-post-read-conversion length)))) | |
41873 | 641 length) |
642 | |
46496 | 643 ;; ucs-tables is preloaded |
644 ;; (defun utf-8-pre-write-conversion (beg end) | |
645 ;; "Semi-dummy pre-write function effectively to autoload ucs-tables." | |
646 ;; ;; Ensure translation table is loaded. | |
647 ;; (require 'ucs-tables) | |
648 ;; ;; Don't do this again. | |
649 ;; (coding-system-put 'mule-utf-8 'pre-write-conversion nil) | |
650 ;; nil) | |
41873 | 651 |
35542 | 652 (make-coding-system |
653 'mule-utf-8 4 ?u | |
654 "UTF-8 encoding for Emacs-supported Unicode characters. | |
46496 | 655 The supported Emacs character sets are the following, plus any other |
656 characters included in the tables `ucs-mule-to-mule-unicode' and | |
657 `utf-8-subst-rev-table': | |
41873 | 658 ascii |
659 eight-bit-control | |
660 eight-bit-graphic | |
661 latin-iso8859-1 | |
662 latin-iso8859-2 | |
663 latin-iso8859-3 | |
664 latin-iso8859-4 | |
665 cyrillic-iso8859-5 | |
666 greek-iso8859-7 | |
667 hebrew-iso8859-8 | |
668 latin-iso8859-9 | |
669 latin-iso8859-14 | |
670 latin-iso8859-15 | |
671 mule-unicode-0100-24ff | |
672 mule-unicode-2500-33ff | |
673 mule-unicode-e000-ffff | |
35542 | 674 |
36243 | 675 Unicode characters out of the ranges U+0000-U+33FF and U+E200-U+FFFF |
46496 | 676 may be decoded into korean-ksc5601, chinese-gb2312, japanese-jisx0208 |
677 \(see user option `utf-8-translate-cjk'); otherwise, sequences of | |
678 eight-bit-control and eight-bit-graphic characters are used to | |
679 preserve their byte sequences, and these are composed to display as a | |
680 single character. Emacs characters that otherwise can't be encoded | |
681 are encoded as U+FFFD." | |
35542 | 682 |
683 '(ccl-decode-mule-utf-8 . ccl-encode-mule-utf-8) | |
684 '((safe-charsets | |
685 ascii | |
686 eight-bit-control | |
687 eight-bit-graphic | |
688 latin-iso8859-1 | |
41873 | 689 latin-iso8859-15 |
690 latin-iso8859-14 | |
691 latin-iso8859-9 | |
692 hebrew-iso8859-8 | |
693 greek-iso8859-7 | |
694 cyrillic-iso8859-5 | |
695 latin-iso8859-4 | |
696 latin-iso8859-3 | |
697 latin-iso8859-2 | |
698 vietnamese-viscii-lower | |
699 vietnamese-viscii-upper | |
700 thai-tis620 | |
701 ipa | |
702 ethiopic | |
703 indian-is13194 | |
704 katakana-jisx0201 | |
705 chinese-sisheng | |
706 lao | |
35542 | 707 mule-unicode-0100-24ff |
708 mule-unicode-2500-33ff | |
709 mule-unicode-e000-ffff) | |
36371
f6bb3ed752b4
(mule-utf-8): Set correct value for valid-codes property.
Kenichi Handa <handa@m17n.org>
parents:
36243
diff
changeset
|
710 (mime-charset . utf-8) |
36423
aa776838b660
(mule-utf-8): Set coding-category property to coding-category-utf-8.
Kenichi Handa <handa@m17n.org>
parents:
36371
diff
changeset
|
711 (coding-category . coding-category-utf-8) |
41873 | 712 (valid-codes (0 . 255)) |
46496 | 713 ;; (pre-write-conversion . utf-8-pre-write-conversion) |
41873 | 714 (post-read-conversion . utf-8-post-read-conversion))) |
35542 | 715 |
716 (define-coding-system-alias 'utf-8 'mule-utf-8) | |
38436
b174db545cfd
Some fixes to follow coding conventions.
Pavel Janík <Pavel@Janik.cz>
parents:
37934
diff
changeset
|
717 |
41873 | 718 ;; I think this needs special private charsets defined for the |
719 ;; untranslated sequences, if it's going to work well. | |
720 | |
721 ;;; (defun utf-8-compose-function (pos to pattern &optional string) | |
722 ;;; (let* ((prop (get-char-property pos 'composition string)) | |
723 ;;; (l (and prop (- (cadr prop) (car prop))))) | |
724 ;;; (cond ((and l (> l (- to pos))) | |
725 ;;; (delete-region pos to)) | |
726 ;;; ((and (> (char-after pos) 224) | |
727 ;;; (< (char-after pos) 256) | |
728 ;;; (save-restriction | |
729 ;;; (narrow-to-region pos to) | |
730 ;;; (utf-8-compose))) | |
731 ;;; t)))) | |
732 | |
733 ;;; (dotimes (i 96) | |
734 ;;; (aset composition-function-table | |
735 ;;; (+ 128 i) | |
736 ;;; `((,(string-as-multibyte "[\200-\237\240-\377]") | |
737 ;;; . utf-8-compose-function)))) | |
738 | |
38436
b174db545cfd
Some fixes to follow coding conventions.
Pavel Janík <Pavel@Janik.cz>
parents:
37934
diff
changeset
|
739 ;;; utf-8.el ends here |