1 ;;; utf-16.el --- UTF-16 encoding/decoding
3 ;; Copyright (C) 2001, 2002 Free Software Foundation, Inc.
5 ;; Author: Dave Love <fx@gnu.org>
6 ;; Keywords: Unicode, UTF-16, i18n
8 ;; This file is part of GNU Emacs.
10 ;; GNU Emacs is free software; you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation; either version 2, or (at your option)
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs; see the file COPYING. If not, write to the
22 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
23 ;; Boston, MA 02111-1307, USA.
27 ;; Support for UTF-16, which is a two-byte encoding (modulo
28 ;; surrogates) of Unicode, written either in little or big endian
29 ;; order and either with or without the leading BOM (a two-byte
30 ;; signature which identifies their byte sex)a.
32 ;; We provides these base coding systems.
35 ;; mule-utf-16le little no
36 ;; mule-utf-16be big no
37 ;; mule-utf-16le-with-signature little yes
38 ;; mule-utf-16be-with-signature big yes
39 ;; mule-utf-16 both yes
41 ;; Note that un-decodable sequences aren't (yet?) preserved as raw
42 ;; bytes, as they are with utf-8, so reading and writing as utf-16 can
47 ;; We end up with trivially different -le and -be versions of most
48 ;; things below, sometimes with commonality abstracted into a let
49 ;; binding for maintenance convenience.
51 ;; We'd need new charsets distinct from ascii and eight-bit-control to
52 ;; deal with untranslated sequences, since we can't otherwise
53 ;; distinguish the bytes, as we can with utf-8.
55 ;; ;; Do a multibyte write for bytes in r3 and r4.
56 ;; ;; Intended for untranslatable utf-16 sequences.
57 ;; (define-ccl-program ccl-mule-utf-16-untrans
60 ;; (r0 = ,(charset-id 'ascii))
62 ;; (r0 = ,(charset-id 'eight-bit-control))
63 ;; (r0 = ,(charset-id 'eight-bit-graphic))))
65 ;; (r0 = ,(charset-id 'ascii))
67 ;; (r0 = ,(charset-id 'eight-bit-control))
68 ;; (r0 = ,(charset-id 'eight-bit-graphic))))
70 ;; "Do a multibyte write for bytes in r3 and r4.
71 ;; First swap them if we're big endian, indicated by r5==0.
72 ;; Intended for untranslatable utf-16 sequences.")
74 ;; Needed in macro expansion, so can't be let-bound. Zapped after use.
76 (defconst utf-16-decode-ucs
77 ;; We have the unicode in r1. Output is charset ID in r0, code
79 `((lookup-integer utf-subst-table-for-decode r1 r3
)
80 (if r7
; got a translation
83 (r0 = ,(charset-id 'ascii
))
85 (r0 = ,(charset-id 'eight-bit-control
))
87 ((r0 = ,(charset-id 'latin-iso8859-1
))
90 ((r0 = ,(charset-id 'mule-unicode-0100-24ff
))
92 (r2 = (((r1 / 96) + 32) << 7))
96 ((r0 = ,(charset-id 'mule-unicode-2500-33ff
))
98 (r2 = (((r1 / 96) + 32) << 7))
101 (if (r1 < #xd800
) ; 2 untranslated bytes
102 ;; ;; Assume this is rare, so don't worry about the
103 ;; ;; overhead of the call.
104 ;; (call mule-utf-16-untrans)
105 ((r0 = ,(charset-id 'mule-unicode-e000-ffff
))
106 (r1 = 15037)) ; U+fffd
107 (if (r1 < #xe000
) ; surrogate
108 ;; ((call mule-utf-16-untrans)
109 ;; (write-multibyte-character r0 r1)
111 ;; (call mule-utf-16-untrans))
113 (r0 = ,(charset-id 'mule-unicode-e000-ffff
))
115 ((r0 = ,(charset-id 'mule-unicode-e000-ffff
))
117 (r2 = (((r1 / 96) + 32) << 7))
119 (r1 += (r2 + 32)))))))))))))
121 (defconst utf-16le-decode-loop
126 (translate-character utf-translation-table-for-decode r0 r1
)
127 (write-multibyte-character r0 r1
)
130 (defconst utf-16be-decode-loop
135 (translate-character utf-translation-table-for-decode r0 r1
)
136 (write-multibyte-character r0 r1
)
141 (define-ccl-program ccl-decode-mule-utf-16le
142 `(2 ; 2 bytes -> 1 to 4 bytes
143 ,utf-16le-decode-loop
)
144 "Decode UTF-16LE (little endian without signature bytes).
145 Basic decoding is done into the charsets ascii, latin-iso8859-1 and
146 mule-unicode-*. Un-representable Unicode characters are decoded as
147 U+fffd. The result is run through the translation-table named
148 `utf-translation-table-for-decode'.")
150 (define-ccl-program ccl-decode-mule-utf-16be
151 `(2 ; 2 bytes -> 1 to 4 bytes
152 ,utf-16be-decode-loop
)
153 "Decode UTF-16BE (big endian without signature bytes).
154 Basic decoding is done into the charsets ascii, latin-iso8859-1 and
155 mule-unicode-*. Un-representable Unicode characters are
156 decoded as U+fffd. The result is run through the translation-table of
157 name `utf-translation-table-for-decode'.")
159 (define-ccl-program ccl-decode-mule-utf-16le-with-signature
162 ,utf-16le-decode-loop
))
163 "Like ccl-decode-utf-16le but skip the first 2-byte BOM.")
165 (define-ccl-program ccl-decode-mule-utf-16be-with-signature
168 ,utf-16be-decode-loop
))
169 "Like ccl-decode-utf-16be but skip the first 2-byte BOM.")
171 (define-ccl-program ccl-decode-mule-utf-16
176 ;; R1 is a BOM for little endian. We keep this character as
177 ;; is temporarily. It is removed by post-read-conversion
180 (write-multibyte-character r0 r1
)
181 ,utf-16le-decode-loop
)
183 ;; R1 is a BOM for big endian, but we can't keep that
184 ;; character in the output because it can't be
185 ;; distinguished with the normal U+FEFF. So, we keep
189 ;; R1 a normal Unicode character.
191 (translate-character utf-translation-table-for-decode r0 r1
)))
192 (write-multibyte-character r0 r1
)
193 ,utf-16be-decode-loop
))))
194 "Like ccl-decode-utf-16be/le but check the first BOM.")
196 (makunbound 'utf-16-decode-ucs
) ; done with it
197 (makunbound 'utf-16le-decode-loop
)
198 (makunbound 'utf-16be-decode-loop
)
201 (defconst utf-16-decode-to-ucs
202 ;; CCL which, given the result of a multibyte read in r0 and r1,
203 ;; sets r0 to the character's Unicode if the charset is one of the
204 ;; basic utf-8 coding system ones. Otherwise set to U+fffd.
205 `(if (r0 == ,(charset-id 'ascii
))
207 (if (r0 == ,(charset-id 'latin-iso8859-1
))
209 (if (r0 == ,(charset-id 'eight-bit-control
))
211 (if (r0 == ,(charset-id 'eight-bit-graphic
))
215 (r3 = ((r1 -
32) * 96))
217 (if (r0 == ,(charset-id 'mule-unicode-0100-24ff
))
219 (if (r0 == ,(charset-id 'mule-unicode-2500-33ff
))
221 (if (r0 == ,(charset-id 'mule-unicode-e000-ffff
))
223 (r0 = #xfffd
))))))))))
225 (defconst utf-16le-encode-loop
227 (read-multibyte-character r0 r1
)
228 (lookup-character utf-subst-table-for-encode r0 r1
)
230 ((translate-character utf-translation-table-for-encode r0 r1
)
231 ,utf-16-decode-to-ucs
))
236 (defconst utf-16be-encode-loop
238 (read-multibyte-character r0 r1
)
239 (lookup-character utf-subst-table-for-encode r0 r1
)
241 ((translate-character utf-translation-table-for-encode r0 r1
)
242 ,utf-16-decode-to-ucs
))
249 (define-ccl-program ccl-encode-mule-utf-16le
251 ,utf-16le-encode-loop
)
252 "Encode to UTF-16LE (little endian without signature).
253 Characters from the charsets ascii, eight-bit-control,
254 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
255 after translation through the translation-table of name
256 `utf-translation-table-for-encode'.
257 Others are encoded as U+FFFD.")
259 (define-ccl-program ccl-encode-mule-utf-16be
261 ,utf-16be-encode-loop
)
262 "Encode to UTF-16BE (big endian without signature).
263 Characters from the charsets ascii, eight-bit-control,
264 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
265 after translation through the translation-table named
266 `utf-translation-table-for-encode'.
267 Others are encoded as U+FFFD.")
269 (define-ccl-program ccl-encode-mule-utf-16le-with-signature
273 ,utf-16le-encode-loop
))
274 "Encode to UTF-16 (little endian with signature).
275 Characters from the charsets ascii, eight-bit-control,
276 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
277 after translation through the translation-table of name
278 `utf-translation-table-for-encode'.
279 Others are encoded as U+FFFD.")
281 (define-ccl-program ccl-encode-mule-utf-16be-with-signature
285 ,utf-16be-encode-loop
))
286 "Encode to UTF-16 (big endian with signature).
287 Characters from the charsets ascii, eight-bit-control,
288 eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
289 after translation through the translation-table named
290 `utf-translation-table-for-encode'.
291 Others are encoded as U+FFFD.")
293 (makunbound 'utf-16-decode-to-ucs
)
294 (makunbound 'utf-16le-encode-loop
)
295 (makunbound 'utf-16be-encode-loop
)
297 (defun mule-utf-16-post-read-conversion (length)
299 (let ((char (following-char)))
300 (cond ((= char
(decode-char 'ucs
#xFFFE
))
302 (setq last-coding-system-used
303 (coding-system-change-text-conversion
304 last-coding-system-used
305 'mule-utf-16le-with-signature
))
306 (setq length
(1- length
)))
307 ((= char
(decode-char 'ucs
#xFFFF
))
309 (setq last-coding-system-used
310 (coding-system-change-text-conversion
311 last-coding-system-used
312 'mule-utf-16be-with-signature
))
313 (setq length
(1- length
)))
315 (setq last-coding-system-used
'mule-utf-16be
)))))
320 It supports Unicode characters of these ranges:
321 U+0000..U+33FF, U+E000..U+FFFF.
322 They correspond to these Emacs character sets:
323 ascii, latin-iso8859-1, mule-unicode-0100-24ff,
324 mule-unicode-2500-33ff, mule-unicode-e000-ffff
326 On decoding (e.g. reading a file), Unicode characters not in the above
327 ranges are decoded as U+FFFD, effectively corrupting the data
328 if they are re-encoded.
330 On encoding (e.g. writing a file), Emacs characters not belonging to
331 any of the character sets listed above are encoded into the byte
332 sequence representing U+FFFD (REPLACEMENT CHARACTER)."))
335 ?u
; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
337 "UTF-16LE encoding for Emacs-supported Unicode characters."
340 '(ccl-decode-mule-utf-16le . ccl-encode-mule-utf-16le
)
345 mule-unicode-0100-24ff
346 mule-unicode-2500-33ff
347 mule-unicode-e000-ffff
)
348 (mime-charset . utf-16le
)
349 (valid-codes (0 .
255))
350 (dependency unify-8859-on-encoding-mode
351 unify-8859-on-decoding-mode
352 utf-fragment-on-decoding
358 "UTF-16BE encoding for Emacs-supported Unicode characters."
361 '(ccl-decode-mule-utf-16be . ccl-encode-mule-utf-16be
)
366 mule-unicode-0100-24ff
367 mule-unicode-2500-33ff
368 mule-unicode-e000-ffff
)
369 (mime-charset . utf-16be
)
370 (valid-codes (0 .
255))
371 (dependency unify-8859-on-encoding-mode
372 unify-8859-on-decoding-mode
373 utf-fragment-on-decoding
377 'mule-utf-16le-with-signature
4 ?u
379 "Little endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
382 '(ccl-decode-mule-utf-16le-with-signature
383 . ccl-encode-mule-utf-16le-with-signature
)
388 mule-unicode-0100-24ff
389 mule-unicode-2500-33ff
390 mule-unicode-e000-ffff
)
391 (coding-category . coding-category-utf-16-le
)
392 (mime-charset . utf-16
)
393 (valid-codes (0 .
255))
394 (dependency unify-8859-on-encoding-mode
395 unify-8859-on-decoding-mode
396 utf-fragment-on-decoding
400 'mule-utf-16be-with-signature
4 ?u
402 "Big endian UTF-16 (with BOM) for Emacs-supported Unicode characters."
405 '(ccl-decode-mule-utf-16be-with-signature
406 . ccl-encode-mule-utf-16be-with-signature
)
411 mule-unicode-0100-24ff
412 mule-unicode-2500-33ff
413 mule-unicode-e000-ffff
)
414 (coding-category . coding-category-utf-16-be
)
415 (mime-charset . utf-16
)
416 (valid-codes (0 .
255))
417 (dependency unify-8859-on-encoding-mode
418 unify-8859-on-decoding-mode
419 utf-fragment-on-decoding
425 "UTF-16 (with or without BOM) for Emacs-supported Unicode characters."
428 '(ccl-decode-mule-utf-16 . ccl-encode-mule-utf-16be-with-signature
)
433 mule-unicode-0100-24ff
434 mule-unicode-2500-33ff
435 mule-unicode-e000-ffff
)
436 (coding-category . coding-category-utf-16-be
)
437 (mime-charset . utf-16
)
438 (valid-codes (0 .
255))
439 (dependency unify-8859-on-encoding-mode
440 unify-8859-on-decoding-mode
441 utf-fragment-on-decoding
443 (post-read-conversion . mule-utf-16-post-read-conversion
)))
446 (define-coding-system-alias 'utf-16le
'mule-utf-16le
)
447 (define-coding-system-alias 'utf-16be
'mule-utf-16be
)
448 (define-coding-system-alias 'utf-16le-with-signature
449 'mule-utf-16le-with-signature
)
450 (define-coding-system-alias 'utf-16be-with-signature
451 'mule-utf-16be-with-signature
)
452 (define-coding-system-alias 'utf-16
'mule-utf-16
)
454 ;; For backward compatibility.
455 (define-coding-system-alias 'mule-utf-16-le
'mule-utf-16le-with-signature
)
456 (define-coding-system-alias 'utf-16-le
'mule-utf-16le-with-signature
)
457 (define-coding-system-alias 'mule-utf-16-be
'mule-utf-16be-with-signature
)
458 (define-coding-system-alias 'utf-16-be
'mule-utf-16be-with-signature
)
460 ;;; utf-16.el ends here