Mercurial > emacs
annotate lisp/international/utf-16.el @ 88123:375f2633d815
New directory
author | Kenichi Handa <handa@m17n.org> |
---|---|
date | Mon, 08 Sep 2003 11:56:09 +0000 |
parents | 695cf19ef79e |
children |
rev | line source |
---|---|
46677 | 1 ;;; utf-16.el --- UTF-16 encoding/decoding |
2 | |
88123 | 3 ;; Copyright (C) 2001, 2002 Free Software Foundation, Inc. |
46677 | 4 |
5 ;; Author: Dave Love <fx@gnu.org> | |
6 ;; Keywords: Unicode, UTF-16, i18n | |
7 | |
8 ;; This file is part of GNU Emacs. | |
9 | |
10 ;; GNU Emacs is free software; you can redistribute it and/or modify | |
11 ;; it under the terms of the GNU General Public License as published by | |
12 ;; the Free Software Foundation; either version 2, or (at your option) | |
13 ;; any later version. | |
14 | |
15 ;; GNU Emacs is distributed in the hope that it will be useful, | |
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 ;; GNU General Public License for more details. | |
19 | |
20 ;; You should have received a copy of the GNU General Public License | |
21 ;; along with GNU Emacs; see the file COPYING. If not, write to the | |
22 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
23 ;; Boston, MA 02111-1307, USA. | |
24 | |
25 ;;; Commentary: | |
26 | |
27 ;; Support for UTF-16, which is a two-byte encoding (modulo | |
88123 | 28 ;; surrogates) of Unicode, written either in little or big endian |
29 ;; order and either with or without the leading BOM (a two-byte | |
30 ;; signature which identifies their byte sex)a. | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
31 ;; |
88123 | 32 ;; We provides these base coding systems. |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
33 ;; name endian BOM |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
34 ;; ---- ------ --- |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
35 ;; mule-utf-16le little no |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
36 ;; mule-utf-16be big no |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
37 ;; mule-utf-16le-with-signature little yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
38 ;; mule-utf-16be-with-signature big yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
39 ;; mule-utf-16 both yes |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
40 ;; |
46677 | 41 ;; Note that un-decodable sequences aren't (yet?) preserved as raw |
42 ;; bytes, as they are with utf-8, so reading and writing as utf-16 can | |
43 ;; corrupt data. | |
44 | |
45 ;;; Code: | |
46 | |
47 ;; We end up with trivially different -le and -be versions of most | |
48 ;; things below, sometimes with commonality abstracted into a let | |
49 ;; binding for maintenance convenience. | |
50 | |
51 ;; We'd need new charsets distinct from ascii and eight-bit-control to | |
52 ;; deal with untranslated sequences, since we can't otherwise | |
53 ;; distinguish the bytes, as we can with utf-8. | |
54 | |
55 ;; ;; Do a multibyte write for bytes in r3 and r4. | |
56 ;; ;; Intended for untranslatable utf-16 sequences. | |
57 ;; (define-ccl-program ccl-mule-utf-16-untrans | |
58 ;; `(0 | |
59 ;; (if (r3 < 128) | |
60 ;; (r0 = ,(charset-id 'ascii)) | |
61 ;; (if (r3 < 160) | |
62 ;; (r0 = ,(charset-id 'eight-bit-control)) | |
63 ;; (r0 = ,(charset-id 'eight-bit-graphic)))) | |
64 ;; (if (r4 < 128) | |
65 ;; (r0 = ,(charset-id 'ascii)) | |
66 ;; (if (r4 < 160) | |
67 ;; (r0 = ,(charset-id 'eight-bit-control)) | |
68 ;; (r0 = ,(charset-id 'eight-bit-graphic)))) | |
69 ;; (r1 = r4))) | |
70 ;; "Do a multibyte write for bytes in r3 and r4. | |
71 ;; First swap them if we're big endian, indicated by r5==0. | |
72 ;; Intended for untranslatable utf-16 sequences.") | |
73 | |
74 ;; Needed in macro expansion, so can't be let-bound. Zapped after use. | |
75 (eval-and-compile | |
76 (defconst utf-16-decode-ucs | |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
77 ;; We have the unicode in r1. Output is charset ID in r0, code |
50483 | 78 ;; point in r1. |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
79 `((lookup-integer utf-subst-table-for-decode r1 r3) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
80 (if r7 ; got a translation |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
81 ((r0 = r1) (r1 = r3)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
82 (if (r1 < 128) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
83 (r0 = ,(charset-id 'ascii)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
84 (if (r1 < 160) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
85 (r0 = ,(charset-id 'eight-bit-control)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
86 (if (r1 < 256) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
87 ((r0 = ,(charset-id 'latin-iso8859-1)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
88 (r1 -= 128)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
89 (if (r1 < #x2500) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
90 ((r0 = ,(charset-id 'mule-unicode-0100-24ff)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
91 (r1 -= #x100) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
92 (r2 = (((r1 / 96) + 32) << 7)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
93 (r1 %= 96) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
94 (r1 += (r2 + 32))) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
95 (if (r1 < #x3400) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
96 ((r0 = ,(charset-id 'mule-unicode-2500-33ff)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
97 (r1 -= #x2500) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
98 (r2 = (((r1 / 96) + 32) << 7)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
99 (r1 %= 96) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
100 (r1 += (r2 + 32))) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
101 (if (r1 < #xd800) ; 2 untranslated bytes |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
102 ;; ;; Assume this is rare, so don't worry about the |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
103 ;; ;; overhead of the call. |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
104 ;; (call mule-utf-16-untrans) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
105 ((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
106 (r1 = 15037)) ; U+fffd |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
107 (if (r1 < #xe000) ; surrogate |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
108 ;; ((call mule-utf-16-untrans) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
109 ;; (write-multibyte-character r0 r1) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
110 ;; (read r3 r4) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
111 ;; (call mule-utf-16-untrans)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
112 ((read r3 r4) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
113 (r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
114 (r1 = 15037)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
115 ((r0 = ,(charset-id 'mule-unicode-e000-ffff)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
116 (r1 -= #xe000) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
117 (r2 = (((r1 / 96) + 32) << 7)) |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
118 (r1 %= 96) |
50483 | 119 (r1 += (r2 + 32))))))))))))) |
120 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
121 (defconst utf-16le-decode-loop |
50483 | 122 `(loop |
123 (read r3 r4) | |
124 (r1 = (r4 <8 r3)) | |
125 ,utf-16-decode-ucs | |
126 (translate-character utf-translation-table-for-decode r0 r1) | |
127 (write-multibyte-character r0 r1) | |
128 (repeat))) | |
129 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
130 (defconst utf-16be-decode-loop |
50483 | 131 `(loop |
132 (read r3 r4) | |
133 (r1 = (r3 <8 r4)) | |
134 ,@utf-16-decode-ucs | |
135 (translate-character utf-translation-table-for-decode r0 r1) | |
136 (write-multibyte-character r0 r1) | |
137 (repeat))) | |
138 | |
139 ) | |
46677 | 140 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
141 (define-ccl-program ccl-decode-mule-utf-16le |
46677 | 142 `(2 ; 2 bytes -> 1 to 4 bytes |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
143 ,utf-16le-decode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
144 "Decode UTF-16LE (little endian without signature bytes). |
46677 | 145 Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
146 mule-unicode-*. Un-representable Unicode characters are decoded as |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
147 U+fffd. The result is run through the translation-table named |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
148 `utf-translation-table-for-decode'.") |
46677 | 149 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
150 (define-ccl-program ccl-decode-mule-utf-16be |
46677 | 151 `(2 ; 2 bytes -> 1 to 4 bytes |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
152 ,utf-16be-decode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
153 "Decode UTF-16BE (big endian without signature bytes). |
46677 | 154 Basic decoding is done into the charsets ascii, latin-iso8859-1 and |
155 mule-unicode-*. Un-representable Unicode characters are | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
156 decoded as U+fffd. The result is run through the translation-table of |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
157 name `utf-translation-table-for-decode'.") |
46677 | 158 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
159 (define-ccl-program ccl-decode-mule-utf-16le-with-signature |
50483 | 160 `(2 |
161 ((read r3 r4) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
162 ,utf-16le-decode-loop)) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
163 "Like ccl-decode-utf-16le but skip the first 2-byte BOM.") |
50483 | 164 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
165 (define-ccl-program ccl-decode-mule-utf-16be-with-signature |
50483 | 166 `(2 |
167 ((read r3 r4) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
168 ,utf-16be-decode-loop)) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
169 "Like ccl-decode-utf-16be but skip the first 2-byte BOM.") |
50483 | 170 |
171 (define-ccl-program ccl-decode-mule-utf-16 | |
172 `(2 | |
173 ((read r3 r4) | |
174 (r1 = (r3 <8 r4)) | |
175 (if (r1 == #xFFFE) | |
176 ;; R1 is a BOM for little endian. We keep this character as | |
177 ;; is temporarily. It is removed by post-read-conversion | |
178 ;; function. | |
179 (,@utf-16-decode-ucs | |
180 (write-multibyte-character r0 r1) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
181 ,utf-16le-decode-loop) |
50483 | 182 ((if (r1 == #xFEFF) |
183 ;; R1 is a BOM for big endian, but we can't keep that | |
184 ;; character in the output because it can't be | |
185 ;; distinguished with the normal U+FEFF. So, we keep | |
186 ;; #xFFFF instead. | |
187 ((r1 = #xFFFF) | |
188 ,@utf-16-decode-ucs) | |
189 ;; R1 a normal Unicode character. | |
190 (,@utf-16-decode-ucs | |
191 (translate-character utf-translation-table-for-decode r0 r1))) | |
192 (write-multibyte-character r0 r1) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
193 ,utf-16be-decode-loop)))) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
194 "Like ccl-decode-utf-16be/le but check the first BOM.") |
50483 | 195 |
46677 | 196 (makunbound 'utf-16-decode-ucs) ; done with it |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
197 (makunbound 'utf-16le-decode-loop) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
198 (makunbound 'utf-16be-decode-loop) |
46677 | 199 |
200 (eval-and-compile | |
201 (defconst utf-16-decode-to-ucs | |
202 ;; CCL which, given the result of a multibyte read in r0 and r1, | |
203 ;; sets r0 to the character's Unicode if the charset is one of the | |
204 ;; basic utf-8 coding system ones. Otherwise set to U+fffd. | |
205 `(if (r0 == ,(charset-id 'ascii)) | |
206 (r0 = r1) | |
207 (if (r0 == ,(charset-id 'latin-iso8859-1)) | |
208 (r0 = (r1 + 128)) | |
209 (if (r0 == ,(charset-id 'eight-bit-control)) | |
210 (r0 = r1) | |
211 (if (r0 == ,(charset-id 'eight-bit-graphic)) | |
212 (r0 = r1) | |
213 ((r2 = (r1 & #x7f)) | |
214 (r1 >>= 7) | |
215 (r3 = ((r1 - 32) * 96)) | |
216 (r3 += (r2 - 32)) | |
217 (if (r0 == ,(charset-id 'mule-unicode-0100-24ff)) | |
218 (r0 = (r3 + #x100)) | |
219 (if (r0 == ,(charset-id 'mule-unicode-2500-33ff)) | |
220 (r0 = (r3 + #x2500)) | |
221 (if (r0 == ,(charset-id 'mule-unicode-e000-ffff)) | |
222 (r0 = (r3 + #xe000)) | |
50483 | 223 (r0 = #xfffd)))))))))) |
224 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
225 (defconst utf-16le-encode-loop |
50483 | 226 `(loop |
227 (read-multibyte-character r0 r1) | |
228 (lookup-character utf-subst-table-for-encode r0 r1) | |
229 (if (r7 == 0) | |
230 ((translate-character utf-translation-table-for-encode r0 r1) | |
231 ,utf-16-decode-to-ucs)) | |
232 (write (r0 & 255)) | |
233 (write (r0 >> 8)) | |
234 (repeat))) | |
235 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
236 (defconst utf-16be-encode-loop |
50483 | 237 `(loop |
238 (read-multibyte-character r0 r1) | |
239 (lookup-character utf-subst-table-for-encode r0 r1) | |
240 (if (r7 == 0) | |
241 ((translate-character utf-translation-table-for-encode r0 r1) | |
242 ,utf-16-decode-to-ucs)) | |
243 (write (r0 >> 8)) | |
244 (write (r0 & 255)) | |
245 (repeat))) | |
246 ) | |
46677 | 247 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
248 |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
249 (define-ccl-program ccl-encode-mule-utf-16le |
46677 | 250 `(1 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
251 ,utf-16le-encode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
252 "Encode to UTF-16LE (little endian without signature). |
46677 | 253 Characters from the charsets ascii, eight-bit-control, |
254 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
255 after translation through the translation-table of name |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
256 `utf-translation-table-for-encode'. |
46677 | 257 Others are encoded as U+FFFD.") |
258 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
259 (define-ccl-program ccl-encode-mule-utf-16be |
46677 | 260 `(1 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
261 ,utf-16be-encode-loop) |
50349
12444cb90785
(ccl-decode-mule-utf-16-le): Don't assume the signature bytes.
Kenichi Handa <handa@m17n.org>
parents:
49598
diff
changeset
|
262 "Encode to UTF-16BE (big endian without signature). |
46677 | 263 Characters from the charsets ascii, eight-bit-control, |
264 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
265 after translation through the translation-table named |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
266 `utf-translation-table-for-encode'. |
46677 | 267 Others are encoded as U+FFFD.") |
268 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
269 (define-ccl-program ccl-encode-mule-utf-16le-with-signature |
50483 | 270 `(1 |
271 ((write #xFF) | |
272 (write #xFE) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
273 ,utf-16le-encode-loop)) |
50483 | 274 "Encode to UTF-16 (little endian with signature). |
275 Characters from the charsets ascii, eight-bit-control, | |
276 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
277 after translation through the translation-table of name | |
278 `utf-translation-table-for-encode'. | |
279 Others are encoded as U+FFFD.") | |
280 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
281 (define-ccl-program ccl-encode-mule-utf-16be-with-signature |
50483 | 282 `(1 |
283 ((write #xFE) | |
284 (write #xFF) | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
285 ,utf-16be-encode-loop)) |
50483 | 286 "Encode to UTF-16 (big endian with signature). |
287 Characters from the charsets ascii, eight-bit-control, | |
288 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded | |
289 after translation through the translation-table named | |
290 `utf-translation-table-for-encode'. | |
291 Others are encoded as U+FFFD.") | |
292 | |
46677 | 293 (makunbound 'utf-16-decode-to-ucs) |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
294 (makunbound 'utf-16le-encode-loop) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
295 (makunbound 'utf-16be-encode-loop) |
50483 | 296 |
297 (defun mule-utf-16-post-read-conversion (length) | |
298 (when (> length 0) | |
299 (let ((char (following-char))) | |
300 (cond ((= char (decode-char 'ucs #xFFFE)) | |
301 (delete-char 1) | |
302 (setq last-coding-system-used | |
303 (coding-system-change-text-conversion | |
304 last-coding-system-used | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
305 'mule-utf-16le-with-signature)) |
50483 | 306 (setq length (1- length))) |
307 ((= char (decode-char 'ucs #xFFFF)) | |
308 (delete-char 1) | |
309 (setq last-coding-system-used | |
310 (coding-system-change-text-conversion | |
311 last-coding-system-used | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
312 'mule-utf-16be-with-signature)) |
50483 | 313 (setq length (1- length))) |
314 (t | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
315 (setq last-coding-system-used 'mule-utf-16be))))) |
50483 | 316 length) |
46677 | 317 |
318 (let ((doc " | |
319 | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
320 It supports Unicode characters of these ranges: |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
321 U+0000..U+33FF, U+E000..U+FFFF. |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
322 They correspond to these Emacs character sets: |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
323 ascii, latin-iso8859-1, mule-unicode-0100-24ff, |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
324 mule-unicode-2500-33ff, mule-unicode-e000-ffff |
46677 | 325 |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
326 On decoding (e.g. reading a file), Unicode characters not in the above |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
327 ranges are decoded as U+FFFD, effectively corrupting the data |
49598
0d8b17d428b5
Trailing whitepace deleted.
Juanma Barranquero <lekktu@gmail.com>
parents:
49203
diff
changeset
|
328 if they are re-encoded. |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
329 |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
330 On encoding (e.g. writing a file), Emacs characters not belonging to |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
331 any of the character sets listed above are encoded into the byte |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
332 sequence representing U+FFFD (REPLACEMENT CHARACTER).")) |
46677 | 333 (make-coding-system |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
334 'mule-utf-16le 4 |
46677 | 335 ?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. |
336 (concat | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
337 "UTF-16LE encoding for Emacs-supported Unicode characters." |
46677 | 338 doc) |
339 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
340 '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le) |
46677 | 341 '((safe-charsets |
342 ascii | |
343 eight-bit-control | |
344 latin-iso8859-1 | |
345 mule-unicode-0100-24ff | |
346 mule-unicode-2500-33ff | |
347 mule-unicode-e000-ffff) | |
348 (mime-charset . utf-16le) | |
349 (valid-codes (0 . 255)) | |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
350 (dependency unify-8859-on-encoding-mode |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
351 unify-8859-on-decoding-mode |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
352 utf-fragment-on-decoding |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
353 utf-translate-cjk))) |
46677 | 354 |
355 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
356 'mule-utf-16be 4 ?u |
46677 | 357 (concat |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
358 "UTF-16BE encoding for Emacs-supported Unicode characters." |
46677 | 359 doc) |
360 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
361 '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be) |
46677 | 362 '((safe-charsets |
363 ascii | |
364 eight-bit-control | |
365 latin-iso8859-1 | |
366 mule-unicode-0100-24ff | |
367 mule-unicode-2500-33ff | |
368 mule-unicode-e000-ffff) | |
369 (mime-charset . utf-16be) | |
50483 | 370 (valid-codes (0 . 255)) |
371 (dependency unify-8859-on-encoding-mode | |
372 unify-8859-on-decoding-mode | |
373 utf-fragment-on-decoding | |
374 utf-translate-cjk))) | |
375 | |
376 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
377 'mule-utf-16le-with-signature 4 ?u |
50483 | 378 (concat |
379 "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters." | |
380 doc) | |
381 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
382 '(ccl-decode-mule-utf-16le-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
383 . ccl-encode-mule-utf-16le-with-signature) |
50483 | 384 '((safe-charsets |
385 ascii | |
386 eight-bit-control | |
387 latin-iso8859-1 | |
388 mule-unicode-0100-24ff | |
389 mule-unicode-2500-33ff | |
390 mule-unicode-e000-ffff) | |
391 (coding-category . coding-category-utf-16-le) | |
392 (mime-charset . utf-16) | |
46677 | 393 (valid-codes (0 . 255)) |
47705
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
394 (dependency unify-8859-on-encoding-mode |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
395 unify-8859-on-decoding-mode |
797c350a7f8c
(utf-16-decode-ucs): Look up
Kenichi Handa <handa@m17n.org>
parents:
47384
diff
changeset
|
396 utf-fragment-on-decoding |
50483 | 397 utf-translate-cjk))) |
398 | |
399 (make-coding-system | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
400 'mule-utf-16be-with-signature 4 ?u |
50483 | 401 (concat |
402 "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters." | |
403 doc) | |
404 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
405 '(ccl-decode-mule-utf-16be-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
406 . ccl-encode-mule-utf-16be-with-signature) |
50483 | 407 '((safe-charsets |
408 ascii | |
409 eight-bit-control | |
410 latin-iso8859-1 | |
411 mule-unicode-0100-24ff | |
412 mule-unicode-2500-33ff | |
413 mule-unicode-e000-ffff) | |
414 (coding-category . coding-category-utf-16-be) | |
415 (mime-charset . utf-16) | |
416 (valid-codes (0 . 255)) | |
417 (dependency unify-8859-on-encoding-mode | |
418 unify-8859-on-decoding-mode | |
419 utf-fragment-on-decoding | |
420 utf-translate-cjk))) | |
421 | |
422 (make-coding-system | |
423 'mule-utf-16 4 ?u | |
424 (concat | |
425 "UTF-16 (with or without BOM) for Emacs-supported Unicode characters." | |
426 doc) | |
427 | |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
428 '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature) |
50483 | 429 '((safe-charsets |
430 ascii | |
431 eight-bit-control | |
432 latin-iso8859-1 | |
433 mule-unicode-0100-24ff | |
434 mule-unicode-2500-33ff | |
435 mule-unicode-e000-ffff) | |
436 (coding-category . coding-category-utf-16-be) | |
437 (mime-charset . utf-16) | |
438 (valid-codes (0 . 255)) | |
439 (dependency unify-8859-on-encoding-mode | |
440 unify-8859-on-decoding-mode | |
441 utf-fragment-on-decoding | |
442 utf-translate-cjk) | |
443 (post-read-conversion . mule-utf-16-post-read-conversion))) | |
444 ) | |
46677 | 445 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
446 (define-coding-system-alias 'utf-16le 'mule-utf-16le) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
447 (define-coding-system-alias 'utf-16be 'mule-utf-16be) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
448 (define-coding-system-alias 'utf-16le-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
449 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
450 (define-coding-system-alias 'utf-16be-with-signature |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
451 'mule-utf-16be-with-signature) |
50483 | 452 (define-coding-system-alias 'utf-16 'mule-utf-16) |
46677 | 453 |
51631
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
454 ;; For backward compatibility. |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
455 (define-coding-system-alias 'mule-utf-16-le 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
456 (define-coding-system-alias 'utf-16-le 'mule-utf-16le-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
457 (define-coding-system-alias 'mule-utf-16-be 'mule-utf-16be-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
458 (define-coding-system-alias 'utf-16-be 'mule-utf-16be-with-signature) |
bc72d6855260
Many name changes: utf-16-{be,le} -> utf-16{be,le}.
Kenichi Handa <handa@m17n.org>
parents:
50483
diff
changeset
|
459 |
46677 | 460 ;;; utf-16.el ends here |