Mercurial > emacs
annotate lisp/international/utf-16.el @ 87498:566965810b9e
*** empty log message ***
author | Magnus Henoch <mange@freemail.hu> |
---|---|
date | Mon, 31 Dec 2007 00:57:51 +0000 |
parents | 2daf9c28b3a4 |
children | b6fdfff4ae81 |
rev | line source |
---|---|
46677 | 1 ;;; utf-16.el --- UTF-16 encoding/decoding |
2 | |
75347 | 3 ;; Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007 |
74544 | 4 ;; Free Software Foundation, Inc. |
75347 | 5 ;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 |
62274 | 6 ;; National Institute of Advanced Industrial Science and Technology (AIST) |
7 ;; Registration Number H14PRO021 | |
46677 | 8 |
9 ;; Author: Dave Love <fx@gnu.org> | |
10 ;; Keywords: Unicode, UTF-16, i18n | |
11 | |
12 ;; This file is part of GNU Emacs. | |
13 | |
14 ;; GNU Emacs is free software; you can redistribute it and/or modify | |
15 ;; it under the terms of the GNU General Public License as published by | |
78310
2daf9c28b3a4
Restore comma mistakenly removed in last change.
Glenn Morris <rgm@gnu.org>
parents:
78301
diff
changeset
|
16 ;; the Free Software Foundation; either version 3, or (at your option) |
46677 | 17 ;; any later version. |
18 | |
19 ;; GNU Emacs is distributed in the hope that it will be useful, | |
20 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
21 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
22 ;; GNU General Public License for more details. | |
23 | |
24 ;; You should have received a copy of the GNU General Public License | |
25 ;; along with GNU Emacs; see the file COPYING. If not, write to the | |
64085 | 26 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, |
27 ;; Boston, MA 02110-1301, USA. | |
46677 | 28 |
29 ;;; Commentary: | |
30 | |
31 ;; Support for UTF-16, which is a two-byte encoding (modulo | |
52285
6c91b4a8b9af
Add mime-text-unsuitable coding system
Dave Love <fx@gnu.org>
parents:
51631
diff
changeset
|
32 ;; surrogates) of Unicode, defined in RFC 2781. It is written either |
6c91b4a8b9af
Add mime-text-unsuitable coding system
Dave Love <fx@gnu.org>
parents:
51631
diff
changeset
|
33 ;; in little or big endian order and either with or without the |
6c91b4a8b9af
Add mime-text-unsuitable coding system
Dave Love <fx@gnu.org>
parents:
51631
diff
changeset
|
34 ;; leading BOM (a two-byte signature which identifies their byte sex). |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
35 ;; |
52285
6c91b4a8b9af
Add mime-text-unsuitable coding system
Dave Love <fx@gnu.org>
parents:
51631
diff
changeset
|
36 ;; We provide these base coding systems. |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
37 ;; name endian BOM |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
38 ;; ---- ------ --- |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
39 ;; mule-utf-16le little no |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
40 ;; mule-utf-16be big no |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
41 ;; mule-utf-16le-with-signature little yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
42 ;; mule-utf-16be-with-signature big yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
43 ;; mule-utf-16 both yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
44 ;; |
46677 | 45 ;; Note that un-decodable sequences aren't (yet?) preserved as raw |
46 ;; bytes, as they are with utf-8, so reading and writing as utf-16 can | |
47 ;; corrupt data. | |
48 | |
49 ;;; Code: | |
50 | |
51 ;; We end up with trivially different -le and -be versions of most | |
52 ;; things below, sometimes with commonality abstracted into a let | |
53 ;; binding for maintenance convenience. | |
54 | |
55 ;; Needed in macro expansion, so can't be let-bound. Zapped after use. | |
56 (eval-and-compile | |
57 (defconst utf-16-decode-ucs | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
58 ;; If r5 is negative, r1 is a Unicode chacter code. Otherise, r5 is |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
59 ;; the first of a surrogate pair and r1 is the second of the pair. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
60 ;; Output is charset ID in r0, code point in r1. R0 may be set to |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
61 ;; -1 in which case a caller should not write out r1. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
62 `((if (r5 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
63 ((r0 = (r1 < #xDC00)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
64 (if ((r1 >= #xE000) | r0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
65 ;; Invalid second code of surrogate pair. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
66 ((r0 = r5) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
67 (call ccl-mule-utf-untrans)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
68 ((r1 -= #xDC00) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
69 (r1 += (((r5 - #xD800) << 10) + #x10000)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
70 (r5 = -1))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
71 (if (r1 < 128) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
72 (r0 = ,(charset-id 'ascii)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
73 ((lookup-integer utf-subst-table-for-decode r1 r3) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
74 (if r7 ; got a translation |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
75 ((r0 = r1) (r1 = r3)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
76 (if (r1 < 160) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
77 (r0 = ,(charset-id 'eight-bit-control)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
78 (if (r1 < 256) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
79 ((r0 = ,(charset-id 'latin-iso8859-1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
80 (r1 -= 128)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
81 (if (r1 < #x2500) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
82 ((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
83 (r1 -= #x100) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
84 (r2 = (((r1 / 96) + 32) << 7)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
85 (r1 %= 96) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
86 (r1 += (r2 + 32))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
87 (if (r1 < #x3400) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
88 ((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
89 (r1 -= #x2500) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
90 (r2 = (((r1 / 96) + 32) << 7)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
91 (r1 %= 96) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
92 (r1 += (r2 + 32))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
93 (if (r1 < #xD800) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
94 ;; We can't have this character. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
95 ((r0 = r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
96 (call ccl-mule-utf-untrans) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
97 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
98 (r0 = -1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
99 (if (r1 < #xDC00) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
100 ;; The first code of a surrogate pair. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
101 ((r5 = r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
102 (r0 = -1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
103 (if (r1 < #xE000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
104 ;; The second code of a surrogate pair, invalid. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
105 ((r0 = r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
106 (call ccl-mule-utf-untrans) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
107 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
108 (r0 = -1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
109 (if (r1 < #x10000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
110 ((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
111 (r1 -= #xE000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
112 (r2 = (((r1 / 96) + 32) << 7)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
113 (r1 %= 96) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
114 (r1 += (r2 + 32))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
115 ;; We can't have this character. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
116 ((r0 = r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
117 (call ccl-mule-utf-untrans) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
118 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
119 (r0 = -1))))))))))))))) |
50483 | 120 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
121 (defconst utf-16le-decode-loop |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
122 `((r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
123 (loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
124 (r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
125 (read r3 r4) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
126 (r1 = (r4 <8 r3)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
127 ,@utf-16-decode-ucs |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
128 (if (r0 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
129 ((translate-character utf-translation-table-for-decode r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
130 (write-multibyte-character r0 r1))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
131 (repeat)))) |
50483 | 132 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
133 (defconst utf-16be-decode-loop |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
134 `((r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
135 (loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
136 (r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
137 (read r3 r4) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
138 (r1 = (r3 <8 r4)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
139 ,@utf-16-decode-ucs |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
140 (if (r0 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
141 ((translate-character utf-translation-table-for-decode r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
142 (write-multibyte-character r0 r1))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
143 (repeat)))) |
50483 | 144 |
145 ) | |
46677 | 146 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
147 (define-ccl-program ccl-decode-mule-utf-16le |
46677 | 148 `(2 ; 2 bytes -> 1 to 4 bytes |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
149 ,utf-16le-decode-loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
150 ((if (r5 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
151 ((r0 = r5) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
152 (call ccl-mule-utf-untrans))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
153 (if (r3 < 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
154 nil |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
155 ((if (r3 < #xA0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
156 (r0 = ,(charset-id 'eight-bit-control)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
157 (r0 = ,(charset-id 'eight-bit-graphic))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
158 (write-multibyte-character r0 r3))))) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
159 "Decode UTF-16LE (little endian without signature bytes). |
46677 | 160 Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
161 mule-unicode-*. Un-representable Unicode characters are decoded as |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
162 U+fffd. The result is run through the translation-table named |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
163 `utf-translation-table-for-decode'.") |
46677 | 164 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
165 (define-ccl-program ccl-decode-mule-utf-16be |
46677 | 166 `(2 ; 2 bytes -> 1 to 4 bytes |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
167 ,utf-16be-decode-loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
168 ((if (r5 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
169 ((r0 = r5) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
170 (call ccl-mule-utf-untrans))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
171 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
172 ((r0 = r3) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
173 (call ccl-mule-utf-untrans))))) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
174 "Decode UTF-16BE (big endian without signature bytes). |
46677 | 175 Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
176 mule-unicode-*. Un-representable Unicode characters are | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
177 decoded as U+fffd. The result is run through the translation-table of |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
178 name `utf-translation-table-for-decode'.") |
46677 | 179 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
180 (define-ccl-program ccl-decode-mule-utf-16le-with-signature |
50483 | 181 `(2 |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
182 ((r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
183 (read r3 r4) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
184 ,@utf-16le-decode-loop) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
185 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
186 ((r0 = r3) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
187 (call ccl-mule-utf-untrans)))) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
188 "Like ccl-decode-utf-16le but skip the first 2-byte BOM.") |
50483 | 189 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
190 (define-ccl-program ccl-decode-mule-utf-16be-with-signature |
50483 | 191 `(2 |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
192 ((r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
193 (read r3 r4) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
194 ,@utf-16be-decode-loop) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
195 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
196 ((r0 = r3) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
197 (call ccl-mule-utf-untrans)))) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
198 "Like ccl-decode-utf-16be but skip the first 2-byte BOM.") |
50483 | 199 |
200 (define-ccl-program ccl-decode-mule-utf-16 | |
201 `(2 | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
202 ((r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
203 (read r3 r4) |
50483 | 204 (r1 = (r3 <8 r4)) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
205 (r5 = -1) |
50483 | 206 (if (r1 == #xFFFE) |
207 ;; R1 is a BOM for little endian. We keep this character as | |
208 ;; is temporarily. It is removed by post-read-conversion | |
209 ;; function. | |
210 (,@utf-16-decode-ucs | |
211 (write-multibyte-character r0 r1) | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
212 ,@utf-16le-decode-loop) |
50483 | 213 ((if (r1 == #xFEFF) |
214 ;; R1 is a BOM for big endian, but we can't keep that | |
215 ;; character in the output because it can't be | |
216 ;; distinguished with the normal U+FEFF. So, we keep | |
217 ;; #xFFFF instead. | |
218 ((r1 = #xFFFF) | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
219 ,@utf-16-decode-ucs |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
220 (write-multibyte-character r0 r1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
221 ;; R1 is a normal Unicode character. |
50483 | 222 (,@utf-16-decode-ucs |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
223 (if (r0 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
224 ((translate-character utf-translation-table-for-decode r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
225 (write-multibyte-character r0 r1))))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
226 ,@utf-16be-decode-loop))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
227 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
228 ((r0 = r3) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
229 (call ccl-mule-utf-untrans)))) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
230 "Like ccl-decode-utf-16be/le but check the first BOM.") |
50483 | 231 |
46677 | 232 (makunbound 'utf-16-decode-ucs) ; done with it |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
233 (makunbound 'utf-16le-decode-loop) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
234 (makunbound 'utf-16be-decode-loop) |
46677 | 235 |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
236 ;; UTF-16 decoder generates an UTF-8 sequence represented by a |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
237 ;; sequence eight-bit-control/graphic chars for an invalid byte (the |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
238 ;; last byte of an odd length source) and an untranslatable character |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
239 ;; (including an invalid surrogate-pair code-point). |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
240 ;; |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
241 ;; This CCL parses that sequence (the first byte is already in r1), |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
242 ;; and if the sequence represents an untranslatable character, it sets |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
243 ;; r1 to the original invalid code or untranslated Unicode character |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
244 ;; code, sets r2 to -1 (to prevent r2 and r3 are written), set2 r5 to |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
245 ;; -1 (to tell the caller that there's no pre-read character). |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
246 ;; |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
247 ;; If the sequence represents an invalid byte, it sets r1 to -1, r2 to |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
248 ;; the byte, sets r3 and r5 to -1. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
249 ;; |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
250 ;; Otherwise, don't change r1, set r2 and r3 to already read |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
251 ;; eight-bit-control/graphic characters (if any), set r5 and r6 to the |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
252 ;; last character that invalidates the UTF-8 form. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
253 ;; |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
254 ;; Note: For UTF-8 validation, we only check if a character is |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
255 ;; eight-bit-control/graphic or not. It may result in incorrect |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
256 ;; handling of random binary data, but such a data can't be encoded by |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
257 ;; UTF-16 anyway. At least, UTF-16 decoder doesn't generate such a |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
258 ;; sequence even if a source contains invalid byte-sequence. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
259 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
260 (define-ccl-program ccl-mule-utf-16-encode-untrans |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
261 `(0 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
262 ((r2 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
263 ;; Read the 2nd byte. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
264 (read-multibyte-character r5 r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
265 (r0 = (r5 != ,(charset-id 'eight-bit-control))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
266 (if ((r5 != ,(charset-id 'eight-bit-graphic)) & r0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
267 ((r2 = r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
268 (r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
269 (r1 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
270 (end))) ; invalid UTF-8 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
271 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
272 (r3 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
273 (r2 = r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
274 (if (r1 <= #xE0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
275 ;; 2-byte UTF-8, i.e. originally an invalid byte. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
276 ((r2 &= #x3F) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
277 (r2 |= ((r1 & #x1F) << 6)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
278 (r1 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
279 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
280 (end))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
281 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
282 ;; Read the 3rd byte. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
283 (read-multibyte-character r5 r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
284 (r0 = (r5 != ,(charset-id 'eight-bit-control))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
285 (if ((r5 != ,(charset-id 'eight-bit-graphic)) & r0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
286 ((end))) ; invalid UTF-8 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
287 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
288 (if (r1 < #xF0) ; valid 3-byte UTF-8 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
289 ((r1 = ((r1 & #x0F) << 12)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
290 (r1 |= ((r2 & #x3F) << 6)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
291 (r1 |= (r6 & #x3F)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
292 (r2 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
293 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
294 (end))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
295 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
296 (r3 = r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
297 ;; Read the 4th byte. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
298 (read-multibyte-character r5 r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
299 (r0 = (r5 != ,(charset-id 'eight-bit-control))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
300 (if ((r5 != ,(charset-id 'eight-bit-graphic)) & r0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
301 (end)) ; livalid UTF-8 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
302 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
303 ;; valid 4-byte UTF-8 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
304 (r1 = ((r1 & #x07) << 18)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
305 (r1 |= ((r2 & #x3F) << 12)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
306 (r1 |= ((r3 & #x3F) << 6)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
307 (r1 |= (r6 & #x3F)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
308 (r2 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
309 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
310 (end)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
311 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
312 (if (r1 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
313 ((write r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
314 (if (r2 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
315 ((write r2) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
316 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
317 (write r3)))))))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
318 |
46677 | 319 (eval-and-compile |
320 (defconst utf-16-decode-to-ucs | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
321 ;; Read a character and set r1 to the corresponding Unicode code. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
322 ;; If r5 is not negative, it means that we have already read a |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
323 ;; character into r5 and r6. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
324 ;; If an invalid eight-bit-control/graphic sequence is found, r2 and |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
325 ;; r3 may contain a byte to written out, r5 and r6 may contain a |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
326 ;; pre-read character. Usually they are set to -1. |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
327 `((if (r5 < 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
328 (read-multibyte-character r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
329 ((r0 = r5) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
330 (r1 = r6) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
331 (r5 = -1))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
332 (lookup-character utf-subst-table-for-encode r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
333 (r2 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
334 (if (r7 > 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
335 (r1 = r0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
336 ((translate-character utf-translation-table-for-encode r0 r1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
337 (if (r0 == ,(charset-id 'ascii)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
338 nil |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
339 (if (r0 == ,(charset-id 'latin-iso8859-1)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
340 (r1 += 128) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
341 (if (r0 == ,(charset-id 'eight-bit-control)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
342 nil |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
343 (if (r0 == ,(charset-id 'eight-bit-graphic)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
344 (call ccl-mule-utf-16-encode-untrans) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
345 ((r2 = ((r1 & #x7f) - 32)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
346 (r3 = ((((r1 >> 7) - 32) * 96) + r2)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
347 (r2 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
348 (r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
349 (if (r0 == ,(charset-id 'mule-unicode-0100-24ff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
350 (r1 = (r3 + #x100)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
351 (if (r0 == ,(charset-id 'mule-unicode-2500-33ff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
352 (r1 = (r3 + #x2500)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
353 (if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
354 (r1 = (r3 + #xe000)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
355 (r1 = #xfffd))))))))))))) |
50483 | 356 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
357 (defconst utf-16le-encode-loop |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
358 `((r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
359 (loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
360 ,@utf-16-decode-to-ucs |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
361 (if (r1 >= #x10000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
362 ((r1 -= #x10000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
363 (r0 = ((r1 >> 10) + #xD800)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
364 (write (r0 & 255)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
365 (write (r0 >> 8)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
366 (r1 = ((r1 & #x3FF) + #xDC00)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
367 (if (r1 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
368 ((write (r1 & 255)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
369 (write (r1 >> 8)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
370 (if (r2 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
371 ((write r2) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
372 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
373 (write r3)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
374 (repeat)))) |
50483 | 375 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
376 (defconst utf-16be-encode-loop |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
377 `((r5 = -1) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
378 (loop |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
379 ,@utf-16-decode-to-ucs |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
380 (if (r1 >= #x10000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
381 ((r1 -= #x10000) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
382 (r0 = ((r1 >> 10) + #xD800)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
383 (write (r0 >> 8)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
384 (write (r0 & 255)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
385 (r1 = ((r1 & #x3FF) + #xDC00)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
386 (if (r1 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
387 ((write (r1 >> 8)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
388 (write (r1 & 255)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
389 (if (r2 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
390 ((write r2) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
391 (if (r3 >= 0) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
392 (write r3)))) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
393 (repeat)))) |
50483 | 394 ) |
46677 | 395 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
396 |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
397 (define-ccl-program ccl-encode-mule-utf-16le |
60305
a878c25f56db
(ccl-encode-mule-utf-16le): Fix
Kenichi Handa <handa@m17n.org>
parents:
56038
diff
changeset
|
398 `(2 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
399 ,utf-16le-encode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
400 "Encode to UTF-16LE (little endian without signature). |
46677 | 401 Characters from the charsets ascii, eight-bit-control, |
402 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
403 after translation through the translation-table of name |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
404 `utf-translation-table-for-encode'. |
46677 | 405 Others are encoded as U+FFFD.") |
406 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
407 (define-ccl-program ccl-encode-mule-utf-16be |
60305
a878c25f56db
(ccl-encode-mule-utf-16le): Fix
Kenichi Handa <handa@m17n.org>
parents:
56038
diff
changeset
|
408 `(2 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
409 ,utf-16be-encode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
410 "Encode to UTF-16BE (big endian without signature). |
46677 | 411 Characters from the charsets ascii, eight-bit-control, |
412 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
413 after translation through the translation-table named |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
414 `utf-translation-table-for-encode'. |
46677 | 415 Others are encoded as U+FFFD.") |
416 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
417 (define-ccl-program ccl-encode-mule-utf-16le-with-signature |
60515
7211619644bb
(ccl-encode-mule-utf-16le-with-signature):
Kenichi Handa <handa@m17n.org>
parents:
60305
diff
changeset
|
418 `(2 |
50483 | 419 ((write #xFF) |
420 (write #xFE) | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
421 ,@utf-16le-encode-loop)) |
50483 | 422 "Encode to UTF-16 (little endian with signature). |
423 Characters from the charsets ascii, eight-bit-control, | |
424 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
425 after translation through the translation-table of name | |
426 `utf-translation-table-for-encode'. | |
427 Others are encoded as U+FFFD.") | |
428 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
429 (define-ccl-program ccl-encode-mule-utf-16be-with-signature |
60515
7211619644bb
(ccl-encode-mule-utf-16le-with-signature):
Kenichi Handa <handa@m17n.org>
parents:
60305
diff
changeset
|
430 `(2 |
50483 | 431 ((write #xFE) |
432 (write #xFF) | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
433 ,@utf-16be-encode-loop)) |
50483 | 434 "Encode to UTF-16 (big endian with signature). |
435 Characters from the charsets ascii, eight-bit-control, | |
436 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
437 after translation through the translation-table named | |
438 `utf-translation-table-for-encode'. | |
439 Others are encoded as U+FFFD.") | |
440 | |
46677 | 441 (makunbound 'utf-16-decode-to-ucs) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
442 (makunbound 'utf-16le-encode-loop) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
443 (makunbound 'utf-16be-encode-loop) |
50483 | 444 |
445 (defun mule-utf-16-post-read-conversion (length) | |
446 (when (> length 0) | |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
447 (setq length (utf-8-post-read-conversion length)) |
50483 | 448 (let ((char (following-char))) |
449 (cond ((= char (decode-char 'ucs #xFFFE)) | |
450 (delete-char 1) | |
451 (setq last-coding-system-used | |
452 (coding-system-change-text-conversion | |
453 last-coding-system-used | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
454 'mule-utf-16le-with-signature)) |
50483 | 455 (setq length (1- length))) |
456 ((= char (decode-char 'ucs #xFFFF)) | |
457 (delete-char 1) | |
458 (setq last-coding-system-used | |
459 (coding-system-change-text-conversion | |
460 last-coding-system-used | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
461 'mule-utf-16be-with-signature)) |
50483 | 462 (setq length (1- length))) |
463 (t | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
464 (setq last-coding-system-used 'mule-utf-16be))))) |
50483 | 465 length) |
46677 | 466 |
467 (let ((doc " | |
468 | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
469 It supports Unicode characters of these ranges: |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
470 U+0000..U+33FF, U+E000..U+FFFF. |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
471 They correspond to these Emacs character sets: |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
472 ascii, latin-iso8859-1, mule-unicode-0100-24ff, |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
473 mule-unicode-2500-33ff, mule-unicode-e000-ffff |
46677 | 474 |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
475 On decoding (e.g. reading a file), Unicode characters not in the above |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
476 ranges are decoded as U+FFFD, effectively corrupting the data |
49598
0d8b17d428b5
Trailing whitepace deleted.
Juanma Barranquero <lekktu@gmail.com>
parents:
49203
diff
changeset
|
477 if they are re-encoded. |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
478 |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
479 On encoding (e.g. writing a file), Emacs characters not belonging to |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
480 any of the character sets listed above are encoded into the byte |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
481 sequence representing U+FFFD (REPLACEMENT CHARACTER).") |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
482 (props `((safe-charsets |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
483 ascii |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
484 eight-bit-control |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
485 eight-bit-graphic |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
486 latin-iso8859-1 |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
487 mule-unicode-0100-24ff |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
488 mule-unicode-2500-33ff |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
489 mule-unicode-e000-ffff |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
490 ,@(if utf-translate-cjk-mode |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
491 utf-translate-cjk-charsets)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
492 (valid-codes (0 . 255)) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
493 (mime-text-unsuitable . t) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
494 (pre-write-conversion . utf-8-pre-write-conversion) |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
495 (dependency unify-8859-on-encoding-mode |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
496 unify-8859-on-decoding-mode |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
497 utf-fragment-on-decoding |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
498 utf-translate-cjk-mode)))) |
46677 | 499 (make-coding-system |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
500 'mule-utf-16le 4 |
46677 | 501 ?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
502 (concat | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
503 "UTF-16LE encoding for Emacs-supported Unicode characters." |
46677 | 504 doc) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
505 '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
506 `(,@props |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
507 (post-read-conversion . utf-8-post-read-conversion) |
64123
4161411b91d0
Declare that all UTF-16-based coding
Kenichi Handa <handa@m17n.org>
parents:
64085
diff
changeset
|
508 (ascii-incompatible . t) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
509 (mime-charset . utf-16le))) |
46677 | 510 |
511 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
512 'mule-utf-16be 4 ?u |
46677 | 513 (concat |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
514 "UTF-16BE encoding for Emacs-supported Unicode characters." |
46677 | 515 doc) |
516 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
517 '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
518 `(,@props |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
519 (post-read-conversion . utf-8-post-read-conversion) |
64123
4161411b91d0
Declare that all UTF-16-based coding
Kenichi Handa <handa@m17n.org>
parents:
64085
diff
changeset
|
520 (ascii-incompatible . t) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
521 (mime-charset . utf-16be))) |
50483 | 522 |
523 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
524 'mule-utf-16le-with-signature 4 ?u |
50483 | 525 (concat |
526 "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters." | |
527 doc) | |
528 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
529 '(ccl-decode-mule-utf-16le-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
530 . ccl-encode-mule-utf-16le-with-signature) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
531 `(,@props |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
532 (post-read-conversion . utf-8-post-read-conversion) |
50483 | 533 (coding-category . coding-category-utf-16-le) |
64123
4161411b91d0
Declare that all UTF-16-based coding
Kenichi Handa <handa@m17n.org>
parents:
64085
diff
changeset
|
534 (ascii-incompatible . t) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
535 (mime-charset . utf-16))) |
50483 | 536 |
537 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
538 'mule-utf-16be-with-signature 4 ?u |
50483 | 539 (concat |
540 "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters." | |
541 doc) | |
542 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
543 '(ccl-decode-mule-utf-16be-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
544 . ccl-encode-mule-utf-16be-with-signature) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
545 `(,@props |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
546 (post-read-conversion . utf-8-post-read-conversion) |
50483 | 547 (coding-category . coding-category-utf-16-be) |
64123
4161411b91d0
Declare that all UTF-16-based coding
Kenichi Handa <handa@m17n.org>
parents:
64085
diff
changeset
|
548 (ascii-incompatible . t) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
549 (mime-charset . utf-16))) |
50483 | 550 |
551 (make-coding-system | |
552 'mule-utf-16 4 ?u | |
553 (concat | |
554 "UTF-16 (with or without BOM) for Emacs-supported Unicode characters." | |
555 doc) | |
556 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
557 '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
558 `(,@props |
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
559 (post-read-conversion . mule-utf-16-post-read-conversion) |
50483 | 560 (coding-category . coding-category-utf-16-be) |
64123
4161411b91d0
Declare that all UTF-16-based coding
Kenichi Handa <handa@m17n.org>
parents:
64085
diff
changeset
|
561 (ascii-incompatible . t) |
56038
b7fe21511efe
(utf-16-decode-ucs): Handle a surrogate
Kenichi Handa <handa@m17n.org>
parents:
55437
diff
changeset
|
562 (mime-charset . utf-16))) |
50483 | 563 ) |
46677 | 564 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
565 (define-coding-system-alias 'utf-16le 'mule-utf-16le) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
566 (define-coding-system-alias 'utf-16be 'mule-utf-16be) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
567 (define-coding-system-alias 'utf-16le-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
568 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
569 (define-coding-system-alias 'utf-16be-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
570 'mule-utf-16be-with-signature) |
50483 | 571 (define-coding-system-alias 'utf-16 'mule-utf-16) |
46677 | 572 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
573 ;; For backward compatibility. |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
574 (define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
575 (define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
576 (define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
577 (define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
578 |
52401 | 579 ;;; arch-tag: 85455d46-d9c9-466d-a6f3-c3582a7367c4 |
46677 | 580 ;;; utf-16.el ends here |