| blob | 8d840fc85fd554f6672a9f1987ee350ff187646f |
1 /* Coding system handler (conversion, detection, and etc).
2 Copyright (C) 1995, 1997, 1998 Electrotechnical Laboratory, JAPAN.
3 Licensed to the Free Software Foundation.
5 This file is part of GNU Emacs.
7 GNU Emacs is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2, or (at your option)
10 any later version.
12 GNU Emacs is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GNU Emacs; see the file COPYING. If not, write to
19 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 Boston, MA 02111-1307, USA. */
22 /*** TABLE OF CONTENTS ***
24 1. Preamble
25 2. Emacs' internal format (emacs-mule) handlers
26 3. ISO2022 handlers
27 4. Shift-JIS and BIG5 handlers
28 5. CCL handlers
29 6. End-of-line handlers
30 7. C library functions
31 8. Emacs Lisp library functions
32 9. Post-amble
34 */
36 /*** GENERAL NOTE on CODING SYSTEM ***
38 Coding system is an encoding mechanism of one or more character
39 sets. Here's a list of coding systems which Emacs can handle. When
40 we say "decode", it means converting some other coding system to
41 Emacs' internal format (emacs-internal), and when we say "encode",
42 it means converting the coding system emacs-mule to some other
43 coding system.
45 0. Emacs' internal format (emacs-mule)
47 Emacs itself holds a multi-lingual character in a buffer and a string
48 in a special format. Details are described in section 2.
50 1. ISO2022
52 The most famous coding system for multiple character sets. X's
53 Compound Text, various EUCs (Extended Unix Code), and coding
54 systems used in Internet communication such as ISO-2022-JP are
55 all variants of ISO2022. Details are described in section 3.
57 2. SJIS (or Shift-JIS or MS-Kanji-Code)
59 A coding system to encode character sets: ASCII, JISX0201, and
60 JISX0208. Widely used for PC's in Japan. Details are described in
61 section 4.
63 3. BIG5
65 A coding system to encode character sets: ASCII and Big5. Widely
66 used by Chinese (mainly in Taiwan and Hong Kong). Details are
67 described in section 4. In this file, when we write "BIG5"
68 (all uppercase), we mean the coding system, and when we write
69 "Big5" (capitalized), we mean the character set.
71 4. Raw text
73 A coding system for a text containing random 8-bit code. Emacs does
74 no code conversion on such a text except for end-of-line format.
76 5. Other
78 If a user wants to read/write a text encoded in a coding system not
79 listed above, he can supply a decoder and an encoder for it in CCL
80 (Code Conversion Language) programs. Emacs executes the CCL program
81 while reading/writing.
83 Emacs represents a coding system by a Lisp symbol that has a property
84 `coding-system'. But, before actually using the coding system, the
85 information about it is set in a structure of type `struct
86 coding_system' for rapid processing. See section 6 for more details.
88 */
90 /*** GENERAL NOTES on END-OF-LINE FORMAT ***
92 How end-of-line of a text is encoded depends on a system. For
93 instance, Unix's format is just one byte of `line-feed' code,
94 whereas DOS's format is two-byte sequence of `carriage-return' and
95 `line-feed' codes. MacOS's format is usually one byte of
96 `carriage-return'.
98 Since text characters encoding and end-of-line encoding are
99 independent, any coding system described above can take
100 any format of end-of-line. So, Emacs has information of format of
101 end-of-line in each coding-system. See section 6 for more details.
103 */
105 /*** GENERAL NOTES on `detect_coding_XXX ()' functions ***
107 These functions check if a text between SRC and SRC_END is encoded
108 in the coding system category XXX. Each returns an integer value in
109 which appropriate flag bits for the category XXX is set. The flag
110 bits are defined in macros CODING_CATEGORY_MASK_XXX. Below is the
111 template of these functions. */
112 #if 0
113 int
116 {
117 ...
118 }
119 #endif
121 /*** GENERAL NOTES on `decode_coding_XXX ()' functions ***
123 These functions decode SRC_BYTES length text at SOURCE encoded in
124 CODING to Emacs' internal format (emacs-mule). The resulting text
125 goes to a place pointed to by DESTINATION, the length of which
126 should not exceed DST_BYTES. These functions set the information of
127 original and decoded texts in the members produced, produced_char,
128 consumed, and consumed_char of the structure *CODING.
130 The return value is an integer (CODING_FINISH_XXX) indicating how
131 the decoding finished.
133 DST_BYTES zero means that source area and destination area are
134 overlapped, which means that we can produce a decoded text until it
135 reaches at the head of not-yet-decoded source text.
137 Below is a template of these functions. */
138 #if 0
143 {
144 ...
145 }
146 #endif
148 /*** GENERAL NOTES on `encode_coding_XXX ()' functions ***
150 These functions encode SRC_BYTES length text at SOURCE of Emacs'
151 internal format (emacs-mule) to CODING. The resulting text goes to
152 a place pointed to by DESTINATION, the length of which should not
153 exceed DST_BYTES. These functions set the information of
154 original and encoded texts in the members produced, produced_char,
155 consumed, and consumed_char of the structure *CODING.
157 The return value is an integer (CODING_FINISH_XXX) indicating how
158 the encoding finished.
160 DST_BYTES zero means that source area and destination area are
161 overlapped, which means that we can produce a decoded text until it
162 reaches at the head of not-yet-decoded source text.
164 Below is a template of these functions. */
165 #if 0
170 {
171 ...
172 }
173 #endif
175 /*** COMMONLY USED MACROS ***/
177 /* The following three macros ONE_MORE_BYTE, TWO_MORE_BYTES, and
178 THREE_MORE_BYTES safely get one, two, and three bytes from the
179 source text respectively. If there are not enough bytes in the
180 source, they jump to `label_end_of_loop'. The caller should set
181 variables `src' and `src_end' to appropriate areas in advance. */
183 #define ONE_MORE_BYTE(c1) \
184 do { \
185 if (src < src_end) \
186 c1 = *src++; \
187 else \
188 goto label_end_of_loop; \
189 } while (0)
191 #define TWO_MORE_BYTES(c1, c2) \
192 do { \
193 if (src + 1 < src_end) \
194 c1 = *src++, c2 = *src++; \
195 else \
196 goto label_end_of_loop; \
197 } while (0)
199 #define THREE_MORE_BYTES(c1, c2, c3) \
200 do { \
201 if (src + 2 < src_end) \
202 c1 = *src++, c2 = *src++, c3 = *src++; \
203 else \
204 goto label_end_of_loop; \
205 } while (0)
207 /* The following three macros DECODE_CHARACTER_ASCII,
208 DECODE_CHARACTER_DIMENSION1, and DECODE_CHARACTER_DIMENSION2 put
209 the multi-byte form of a character of each class at the place
210 pointed by `dst'. The caller should set the variable `dst' to
211 point to an appropriate area and the variable `coding' to point to
212 the coding-system of the currently decoding text in advance. */
214 /* Decode one ASCII character C. */
216 #define DECODE_CHARACTER_ASCII(c) \
217 do { \
218 if (COMPOSING_P (coding->composing)) \
219 { \
220 *dst++ = 0xA0, *dst++ = (c) | 0x80; \
221 coding->composed_chars++; \
222 if (((c) | 0x80) < 0xA0) \
223 coding->fake_multibyte = 1; \
224 } \
225 else \
226 { \
227 /* If ASCII charset is invoked to GR, \
229 *dst++ = (c) & 0x7F; \
230 coding->produced_char++; \
231 } \
232 } while (0)
234 /* Decode one DIMENSION1 character whose charset is CHARSET and whose
235 position-code is C. */
237 #define DECODE_CHARACTER_DIMENSION1(charset, c) \
238 do { \
239 unsigned char leading_code = CHARSET_LEADING_CODE_BASE (charset); \
240 if (COMPOSING_P (coding->composing)) \
241 { \
242 *dst++ = leading_code + 0x20; \
243 coding->composed_chars++; \
244 } \
245 else \
246 { \
247 *dst++ = leading_code; \
248 coding->produced_char++; \
249 } \
250 if (leading_code = CHARSET_LEADING_CODE_EXT (charset)) \
251 *dst++ = leading_code; \
252 *dst++ = (c) | 0x80; \
253 if (((c) | 0x80) < 0xA0) \
254 coding->fake_multibyte = 1; \
255 } while (0)
257 /* Decode one DIMENSION2 character whose charset is CHARSET and whose
258 position-codes are C1 and C2. */
260 #define DECODE_CHARACTER_DIMENSION2(charset, c1, c2) \
261 do { \
262 DECODE_CHARACTER_DIMENSION1 (charset, c1); \
263 *dst++ = (c2) | 0x80; \
264 if (((c2) | 0x80) < 0xA0) \
265 coding->fake_multibyte = 1; \
266 } while (0)
268 \f
269 /*** 1. Preamble ***/
271 #ifdef emacs
272 #include <config.h>
273 #endif
275 #include <stdio.h>
277 #ifdef emacs
293 Lisp_Object Qbuffer_file_coding_system;
296 Lisp_Object Qcoding_system_history;
297 Lisp_Object Qsafe_charsets;
298 Lisp_Object Qvalid_codes;
303 Lisp_Object Qtarget_idx;
305 Lisp_Object Vselect_safe_coding_system_function;
307 /* Mnemonic string for each format of end-of-line. */
309 /* Mnemonic string to indicate format of end-of-line is not yet
310 decided. */
311 Lisp_Object eol_mnemonic_undecided;
313 /* Format of end-of-line decided by system. This is CODING_EOL_LF on
314 Unix, CODING_EOL_CRLF on DOS/Windows, and CODING_EOL_CR on Mac. */
317 #ifdef emacs
323 /* Coding system emacs-mule and raw-text are for converting only
324 end-of-line format. */
327 /* Coding-systems are handed between Emacs Lisp programs and C internal
328 routines by the following three variables. */
329 /* Coding-system for reading files and receiving data from process. */
330 Lisp_Object Vcoding_system_for_read;
331 /* Coding-system for writing files and sending data to process. */
332 Lisp_Object Vcoding_system_for_write;
333 /* Coding-system actually used in the latest I/O. */
334 Lisp_Object Vlast_coding_system_used;
336 /* A vector of length 256 which contains information about special
337 Latin codes (especially for dealing with Microsoft codes). */
338 Lisp_Object Vlatin_extra_code_table;
340 /* Flag to inhibit code conversion of end-of-line format. */
343 /* Flag to make buffer-file-coding-system inherit from process-coding. */
346 /* Coding system to be used to encode text for terminal display. */
349 /* Coding system to be used to encode text for terminal display when
350 terminal coding system is nil. */
353 /* Coding system of what is sent from terminal keyboard. */
356 /* Default coding system to be used to write a file. */
359 Lisp_Object Vfile_coding_system_alist;
360 Lisp_Object Vprocess_coding_system_alist;
361 Lisp_Object Vnetwork_coding_system_alist;
363 Lisp_Object Vlocale_coding_system;
369 /* List of symbols `coding-category-xxx' ordered by priority. */
370 Lisp_Object Vcoding_category_list;
372 /* Table of coding categories (Lisp symbols). */
373 Lisp_Object Vcoding_category_table;
375 /* Table of names of symbol for each coding-category. */
388 "coding-category-binary"
389 };
391 /* Table of pointers to coding systems corresponding to each coding
392 categories. */
395 /* Table of coding category masks. Nth element is a mask for a coding
396 cateogry of which priority is Nth. */
397 static
400 /* Flag to tell if we look up translation table on character code
401 conversion. */
402 Lisp_Object Venable_character_translation;
403 /* Standard translation table to look up on decoding (reading). */
404 Lisp_Object Vstandard_translation_table_for_decode;
405 /* Standard translation table to look up on encoding (writing). */
406 Lisp_Object Vstandard_translation_table_for_encode;
408 Lisp_Object Qtranslation_table;
409 Lisp_Object Qtranslation_table_id;
410 Lisp_Object Qtranslation_table_for_decode;
411 Lisp_Object Qtranslation_table_for_encode;
413 /* Alist of charsets vs revision number. */
414 Lisp_Object Vcharset_revision_alist;
416 /* Default coding systems used for process I/O. */
417 Lisp_Object Vdefault_process_coding_system;
419 /* Global flag to tell that we can't call post-read-conversion and
420 pre-write-conversion functions. Usually the value is zero, but it
421 is set to 1 temporarily while such functions are running. This is
422 to avoid infinite recursive call. */
425 \f
426 /*** 2. Emacs internal format (emacs-mule) handlers ***/
428 /* Emacs' internal format for encoding multiple character sets is a
429 kind of multi-byte encoding, i.e. characters are encoded by
430 variable-length sequences of one-byte codes. ASCII characters
431 and control characters (e.g. `tab', `newline') are represented by
432 one-byte sequences which are their ASCII codes, in the range 0x00
433 through 0x7F. The other characters are represented by a sequence
434 of `base leading-code', optional `extended leading-code', and one
435 or two `position-code's. The length of the sequence is determined
436 by the base leading-code. Leading-code takes the range 0x80
437 through 0x9F, whereas extended leading-code and position-code take
438 the range 0xA0 through 0xFF. See `charset.h' for more details
439 about leading-code and position-code.
441 There's one exception to this rule. Special leading-code
442 `leading-code-composition' denotes that the following several
443 characters should be composed into one character. Leading-codes of
444 components (except for ASCII) are added 0x20. An ASCII character
445 component is represented by a 2-byte sequence of `0xA0' and
446 `ASCII-code + 0x80'. See also the comments in `charset.h' for the
447 details of composite character. Hence, we can summarize the code
448 range as follows:
450 --- CODE RANGE of Emacs' internal format ---
451 (character set) (range)
452 ASCII 0x00 .. 0x7F
453 ELSE (1st byte) 0x80 .. 0x9F
454 (rest bytes) 0xA0 .. 0xFF
455 ---------------------------------------------
457 */
461 /* Go to the next statement only if *SRC is accessible and the code is
462 greater than 0xA0. */
463 #define CHECK_CODE_RANGE_A0_FF \
464 do { \
465 if (src >= src_end) \
466 goto label_end_of_switch; \
467 else if (*src++ < 0xA0) \
468 return 0; \
469 } while (0)
471 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
472 Check if a text is encoded in Emacs' internal format. If it is,
473 return CODING_CATEGORY_MASK_EMACS_MULE, else return 0. */
475 int
478 {
483 {
487 {
490 else
492 }
495 {
510 CHECK_CODE_RANGE_A0_FF;
511 else
516 CHECK_CODE_RANGE_A0_FF;
517 /* fall down to check it two more times ... */
520 CHECK_CODE_RANGE_A0_FF;
521 /* fall down to check it one more time ... */
524 CHECK_CODE_RANGE_A0_FF;
528 label_end_of_switch:
530 }
531 }
533 }
535 \f
536 /*** 3. ISO2022 handlers ***/
538 /* The following note describes the coding system ISO2022 briefly.
539 Since the intention of this note is to help understand the
540 functions in this file, some parts are NOT ACCURATE or OVERLY
541 SIMPLIFIED. For thorough understanding, please refer to the
542 original document of ISO2022.
544 ISO2022 provides many mechanisms to encode several character sets
545 in 7-bit and 8-bit environments. For 7-bite environments, all text
546 is encoded using bytes less than 128. This may make the encoded
547 text a little bit longer, but the text passes more easily through
548 several gateways, some of which strip off MSB (Most Signigant Bit).
550 There are two kinds of character sets: control character set and
551 graphic character set. The former contains control characters such
552 as `newline' and `escape' to provide control functions (control
553 functions are also provided by escape sequences). The latter
554 contains graphic characters such as 'A' and '-'. Emacs recognizes
555 two control character sets and many graphic character sets.
557 Graphic character sets are classified into one of the following
558 four classes, according to the number of bytes (DIMENSION) and
559 number of characters in one dimension (CHARS) of the set:
560 - DIMENSION1_CHARS94
561 - DIMENSION1_CHARS96
562 - DIMENSION2_CHARS94
563 - DIMENSION2_CHARS96
565 In addition, each character set is assigned an identification tag,
566 unique for each set, called "final character" (denoted as <F>
567 hereafter). The <F> of each character set is decided by ECMA(*)
568 when it is registered in ISO. The code range of <F> is 0x30..0x7F
569 (0x30..0x3F are for private use only).
571 Note (*): ECMA = European Computer Manufacturers Association
573 Here are examples of graphic character set [NAME(<F>)]:
574 o DIMENSION1_CHARS94 -- ASCII('B'), right-half-of-JISX0201('I'), ...
575 o DIMENSION1_CHARS96 -- right-half-of-ISO8859-1('A'), ...
576 o DIMENSION2_CHARS94 -- GB2312('A'), JISX0208('B'), ...
577 o DIMENSION2_CHARS96 -- none for the moment
579 A code area (1 byte=8 bits) is divided into 4 areas, C0, GL, C1, and GR.
580 C0 [0x00..0x1F] -- control character plane 0
581 GL [0x20..0x7F] -- graphic character plane 0
582 C1 [0x80..0x9F] -- control character plane 1
583 GR [0xA0..0xFF] -- graphic character plane 1
585 A control character set is directly designated and invoked to C0 or
586 C1 by an escape sequence. The most common case is that:
587 - ISO646's control character set is designated/invoked to C0, and
588 - ISO6429's control character set is designated/invoked to C1,
589 and usually these designations/invocations are omitted in encoded
590 text. In a 7-bit environment, only C0 can be used, and a control
591 character for C1 is encoded by an appropriate escape sequence to
592 fit into the environment. All control characters for C1 are
593 defined to have corresponding escape sequences.
595 A graphic character set is at first designated to one of four
596 graphic registers (G0 through G3), then these graphic registers are
597 invoked to GL or GR. These designations and invocations can be
598 done independently. The most common case is that G0 is invoked to
599 GL, G1 is invoked to GR, and ASCII is designated to G0. Usually
600 these invocations and designations are omitted in encoded text.
601 In a 7-bit environment, only GL can be used.
603 When a graphic character set of CHARS94 is invoked to GL, codes
604 0x20 and 0x7F of the GL area work as control characters SPACE and
605 DEL respectively, and codes 0xA0 and 0xFF of the GR area should not
606 be used.
608 There are two ways of invocation: locking-shift and single-shift.
609 With locking-shift, the invocation lasts until the next different
610 invocation, whereas with single-shift, the invocation affects the
611 following character only and doesn't affect the locking-shift
612 state. Invocations are done by the following control characters or
613 escape sequences:
615 ----------------------------------------------------------------------
616 abbrev function cntrl escape seq description
617 ----------------------------------------------------------------------
618 SI/LS0 (shift-in) 0x0F none invoke G0 into GL
619 SO/LS1 (shift-out) 0x0E none invoke G1 into GL
620 LS2 (locking-shift-2) none ESC 'n' invoke G2 into GL
621 LS3 (locking-shift-3) none ESC 'o' invoke G3 into GL
622 LS1R (locking-shift-1 right) none ESC '~' invoke G1 into GR (*)
623 LS2R (locking-shift-2 right) none ESC '}' invoke G2 into GR (*)
624 LS3R (locking-shift 3 right) none ESC '|' invoke G3 into GR (*)
625 SS2 (single-shift-2) 0x8E ESC 'N' invoke G2 for one char
626 SS3 (single-shift-3) 0x8F ESC 'O' invoke G3 for one char
627 ----------------------------------------------------------------------
628 (*) These are not used by any known coding system.
630 Control characters for these functions are defined by macros
631 ISO_CODE_XXX in `coding.h'.
633 Designations are done by the following escape sequences:
634 ----------------------------------------------------------------------
635 escape sequence description
636 ----------------------------------------------------------------------
637 ESC '(' <F> designate DIMENSION1_CHARS94<F> to G0
638 ESC ')' <F> designate DIMENSION1_CHARS94<F> to G1
639 ESC '*' <F> designate DIMENSION1_CHARS94<F> to G2
640 ESC '+' <F> designate DIMENSION1_CHARS94<F> to G3
641 ESC ',' <F> designate DIMENSION1_CHARS96<F> to G0 (*)
642 ESC '-' <F> designate DIMENSION1_CHARS96<F> to G1
643 ESC '.' <F> designate DIMENSION1_CHARS96<F> to G2
644 ESC '/' <F> designate DIMENSION1_CHARS96<F> to G3
645 ESC '$' '(' <F> designate DIMENSION2_CHARS94<F> to G0 (**)
646 ESC '$' ')' <F> designate DIMENSION2_CHARS94<F> to G1
647 ESC '$' '*' <F> designate DIMENSION2_CHARS94<F> to G2
648 ESC '$' '+' <F> designate DIMENSION2_CHARS94<F> to G3
649 ESC '$' ',' <F> designate DIMENSION2_CHARS96<F> to G0 (*)
650 ESC '$' '-' <F> designate DIMENSION2_CHARS96<F> to G1
651 ESC '$' '.' <F> designate DIMENSION2_CHARS96<F> to G2
652 ESC '$' '/' <F> designate DIMENSION2_CHARS96<F> to G3
653 ----------------------------------------------------------------------
655 In this list, "DIMENSION1_CHARS94<F>" means a graphic character set
656 of dimension 1, chars 94, and final character <F>, etc...
658 Note (*): Although these designations are not allowed in ISO2022,
659 Emacs accepts them on decoding, and produces them on encoding
660 CHARS96 character sets in a coding system which is characterized as
661 7-bit environment, non-locking-shift, and non-single-shift.
663 Note (**): If <F> is '@', 'A', or 'B', the intermediate character
664 '(' can be omitted. We refer to this as "short-form" hereafter.
666 Now you may notice that there are a lot of ways for encoding the
667 same multilingual text in ISO2022. Actually, there exist many
668 coding systems such as Compound Text (used in X11's inter client
669 communication, ISO-2022-JP (used in Japanese internet), ISO-2022-KR
670 (used in Korean internet), EUC (Extended UNIX Code, used in Asian
671 localized platforms), and all of these are variants of ISO2022.
673 In addition to the above, Emacs handles two more kinds of escape
674 sequences: ISO6429's direction specification and Emacs' private
675 sequence for specifying character composition.
677 ISO6429's direction specification takes the following form:
678 o CSI ']' -- end of the current direction
679 o CSI '0' ']' -- end of the current direction
680 o CSI '1' ']' -- start of left-to-right text
681 o CSI '2' ']' -- start of right-to-left text
682 The control character CSI (0x9B: control sequence introducer) is
683 abbreviated to the escape sequence ESC '[' in a 7-bit environment.
685 Character composition specification takes the following form:
686 o ESC '0' -- start character composition
687 o ESC '1' -- end character composition
688 Since these are not standard escape sequences of any ISO standard,
689 the use of them for these meaning is restricted to Emacs only. */
693 #define CHARSET_OK(idx, charset) \
694 (coding_system_table[idx] \
695 && (coding_system_table[idx]->safe_charsets[charset] \
696 || (CODING_SPEC_ISO_REQUESTED_DESIGNATION \
697 (coding_system_table[idx], charset) \
698 != CODING_SPEC_ISO_NO_REQUESTED_DESIGNATION)))
700 #define SHIFT_OUT_OK(idx) \
701 (CODING_SPEC_ISO_INITIAL_DESIGNATION (coding_system_table[idx], 1) >= 0)
703 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
704 Check if a text is encoded in ISO2022. If it is, returns an
705 integer in which appropriate flag bits any of:
706 CODING_CATEGORY_MASK_ISO_7
707 CODING_CATEGORY_MASK_ISO_7_TIGHT
708 CODING_CATEGORY_MASK_ISO_8_1
709 CODING_CATEGORY_MASK_ISO_8_2
710 CODING_CATEGORY_MASK_ISO_7_ELSE
711 CODING_CATEGORY_MASK_ISO_8_ELSE
712 are set. If a code which should never appear in ISO2022 is found,
713 returns 0. */
715 int
718 {
726 {
729 {
736 {
737 /* Designation sequence for a charset of dimension 1. */
743 /* Invalid designation sequence. Just ignore. */
746 }
748 {
749 /* Designation sequence for a charset of dimension 2. */
754 /* Designation for JISX0208.1978, GB2312, or JISX0208. */
757 {
763 /* Invalid designation sequence. Just ignore. */
766 }
767 else
768 /* Invalid designation sequence. Just ignore. */
770 }
772 {
773 /* ESC <Fe> for SS2 or SS3. */
776 }
778 /* ESC <Fp> for start/end composition. Just ignore. */
780 else
781 /* Invalid escape sequence. Just ignore. */
784 /* We found a valid designation sequence for CHARSET. */
788 else
792 else
796 else
800 else
810 {
811 /* Locking shift out. */
814 }
820 {
821 /* Locking shift in. */
824 }
831 {
835 {
843 }
846 {
853 }
856 }
861 {
864 }
866 {
870 {
881 }
882 else
884 }
885 else
886 {
889 mask &= ~(CODING_CATEGORY_MASK_ISO_7BIT
892 /* Check the length of succeeding codes of the range
893 0xA0..0FF. If the byte length is odd, we exclude
894 CODING_CATEGORY_MASK_ISO_8_2. We can check this only
895 when we are not single shifting. */
897 {
899 src++;
902 else
904 }
905 }
907 }
908 }
911 }
913 /* Decode a character of which charset is CHARSET and the 1st position
914 code is C1. If dimension of CHARSET is 2, the 2nd position code is
915 fetched from SRC and set to C2. If CHARSET is negative, it means
916 that we are decoding ill formed text, and what we can do is just to
917 read C1 as is. */
919 #define DECODE_ISO_CHARACTER(charset, c1) \
920 do { \
921 int c_alt, charset_alt = (charset); \
922 if (COMPOSING_HEAD_P (coding->composing)) \
923 { \
924 *dst++ = LEADING_CODE_COMPOSITION; \
925 if (COMPOSING_WITH_RULE_P (coding->composing)) \
927 *dst++ = 0xFF; \
928 coding->composing += 2; \
929 } \
930 if (charset_alt >= 0) \
931 { \
932 if (CHARSET_DIMENSION (charset_alt) == 2) \
933 { \
934 ONE_MORE_BYTE (c2); \
935 if (iso_code_class[(c2) & 0x7F] != ISO_0x20_or_0x7F \
936 && iso_code_class[(c2) & 0x7F] != ISO_graphic_plane_0) \
937 { \
938 src--; \
939 charset_alt = CHARSET_ASCII; \
940 } \
941 } \
942 if (!NILP (translation_table) \
943 && ((c_alt = translate_char (translation_table, \
944 -1, charset_alt, c1, c2)) >= 0)) \
945 SPLIT_CHAR (c_alt, charset_alt, c1, c2); \
946 } \
947 if (charset_alt == CHARSET_ASCII || charset_alt < 0) \
948 DECODE_CHARACTER_ASCII (c1); \
949 else if (CHARSET_DIMENSION (charset_alt) == 1) \
950 DECODE_CHARACTER_DIMENSION1 (charset_alt, c1); \
951 else \
952 DECODE_CHARACTER_DIMENSION2 (charset_alt, c1, c2); \
953 if (COMPOSING_WITH_RULE_P (coding->composing)) \
955 coding->composing = COMPOSING_WITH_RULE_RULE; \
956 } while (0)
958 /* Set designation state into CODING. */
959 #define DECODE_DESIGNATION(reg, dimension, chars, final_char) \
960 do { \
961 int charset; \
962 \
964 goto label_invalid_code; \
965 charset = ISO_CHARSET_TABLE (make_number (dimension), \
966 make_number (chars), \
967 make_number (final_char)); \
968 if (charset >= 0 \
969 && (CODING_SPEC_ISO_REQUESTED_DESIGNATION (coding, charset) == reg \
970 || coding->safe_charsets[charset])) \
971 { \
972 if (coding->spec.iso2022.last_invalid_designation_register == 0 \
973 && reg == 0 \
974 && charset == CHARSET_ASCII) \
975 { \
976 /* We should insert this designation sequence as is so \
978 coding->spec.iso2022.last_invalid_designation_register = -1; \
979 goto label_invalid_code; \
980 } \
981 coding->spec.iso2022.last_invalid_designation_register = -1; \
982 if ((coding->mode & CODING_MODE_DIRECTION) \
983 && CHARSET_REVERSE_CHARSET (charset) >= 0) \
984 charset = CHARSET_REVERSE_CHARSET (charset); \
985 CODING_SPEC_ISO_DESIGNATION (coding, reg) = charset; \
986 } \
987 else \
988 { \
989 coding->spec.iso2022.last_invalid_designation_register = reg; \
990 goto label_invalid_code; \
991 } \
992 } while (0)
994 /* Return 0 if there's a valid composing sequence starting at SRC and
995 ending before SRC_END, else return -1. */
997 int
1001 {
1005 {
1022 {
1030 }
1031 else
1033 }
1035 /* We have not found the sequence "ESC 1". */
1037 }
1039 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". */
1041 int
1046 {
1051 /* Since the maximum bytes produced by each loop is 7, we subtract 6
1052 from DST_END to assure that overflow checking is necessary only
1053 at the head of loop. */
1056 /* Charsets invoked to graphic plane 0 and 1 respectively. */
1059 Lisp_Object translation_table
1071 {
1072 /* SRC_BASE remembers the start position in source in each loop.
1073 The loop will be exited when there's not enough source text
1074 to analyze long escape sequence or 2-byte code (within macros
1075 ONE_MORE_BYTE or TWO_MORE_BYTES). In that case, SRC is reset
1076 to SRC_BASE before exiting. */
1081 {
1085 {
1086 /* This is SPACE or DEL. */
1090 }
1091 /* This is a graphic character, we fall down ... */
1095 {
1096 /* This is a composition rule. */
1099 }
1100 else
1108 /* This is a graphic character, we fall down ... */
1113 else
1118 /* All ISO2022 control characters in this class have the
1119 same representation in Emacs internal format. */
1124 {
1127 }
1138 {
1142 else
1143 {
1145 {
1148 }
1149 src--;
1151 }
1152 }
1153 else
1177 /* SS2 is handled as an escape sequence of ESC 'N' */
1184 /* SS2 is handled as an escape sequence of ESC 'O' */
1189 /* CSI is handled as an escape sequence of ESC '[' ... */
1195 label_escape_sequence:
1196 /* Escape sequences handled by Emacs are invocation,
1197 designation, direction specification, and character
1198 composition specification. */
1200 {
1217 or JISX0208.1980 */
1219 }
1224 }
1229 }
1230 else
1269 /* Before processing composing, we must be sure that all
1270 characters being composed are supported by CODING.
1271 If not, we must give up composing. */
1273 {
1274 /* We are looking at a valid composition sequence. */
1276 ? COMPOSING_NO_RULE_HEAD
1279 }
1280 else
1281 {
1285 }
1290 {
1295 }
1298 {
1300 {
1304 /* Only one character is in the composing
1305 sequence. Make it a normal character. */
1307 dst = (this_char_start
1316 }
1318 }
1325 /* For the moment, nested direction is not supported.
1326 So, `coding->mode & CODING_MODE_DIRECTION' zero means
1327 left-to-right, and nozero means right-to-left. */
1330 {
1339 else
1347 else
1353 }
1363 }
1368 }
1369 else
1370 {
1372 }
1373 }
1374 /* We must update these variables now. */
1379 label_invalid_code:
1383 }
1386 label_end_of_loop:
1388 label_end_of_loop_2:
1391 }
1394 {
1399 {
1400 /* This is the last block of the text to be decoded. We had
1401 better just flush out all remaining codes in the text
1402 although they are not valid characters. */
1410 }
1411 }
1416 }
1418 /* ISO2022 encoding stuff. */
1420 /*
1421 It is not enough to say just "ISO2022" on encoding, we have to
1422 specify more details. In Emacs, each coding system of ISO2022
1423 variant has the following specifications:
1424 1. Initial designation to G0 thru G3.
1425 2. Allows short-form designation?
1426 3. ASCII should be designated to G0 before control characters?
1427 4. ASCII should be designated to G0 at end of line?
1428 5. 7-bit environment or 8-bit environment?
1429 6. Use locking-shift?
1430 7. Use Single-shift?
1431 And the following two are only for Japanese:
1432 8. Use ASCII in place of JIS0201-1976-Roman?
1433 9. Use JISX0208-1983 in place of JISX0208-1978?
1434 These specifications are encoded in `coding->flags' as flag bits
1435 defined by macros CODING_FLAG_ISO_XXX. See `coding.h' for more
1436 details.
1437 */
1439 /* Produce codes (escape sequence) for designating CHARSET to graphic
1440 register REG. If <final-char> of CHARSET is '@', 'A', or 'B' and
1441 the coding system CODING allows, produce designation sequence of
1442 short-form. */
1444 #define ENCODE_DESIGNATION(charset, reg, coding) \
1445 do { \
1446 unsigned char final_char = CHARSET_ISO_FINAL_CHAR (charset); \
1449 int revision = CODING_SPEC_ISO_REVISION_NUMBER(coding, charset); \
1450 if (revision < 255) \
1451 { \
1452 *dst++ = ISO_CODE_ESC; \
1455 } \
1456 *dst++ = ISO_CODE_ESC; \
1457 if (CHARSET_DIMENSION (charset) == 1) \
1458 { \
1459 if (CHARSET_CHARS (charset) == 94) \
1460 *dst++ = (unsigned char) (intermediate_char_94[reg]); \
1461 else \
1462 *dst++ = (unsigned char) (intermediate_char_96[reg]); \
1463 } \
1464 else \
1465 { \
1467 if (CHARSET_CHARS (charset) == 94) \
1468 { \
1469 if (! (coding->flags & CODING_FLAG_ISO_SHORT_FORM) \
1470 || reg != 0 \
1472 *dst++ = (unsigned char) (intermediate_char_94[reg]); \
1473 } \
1474 else \
1475 *dst++ = (unsigned char) (intermediate_char_96[reg]); \
1476 } \
1477 *dst++ = final_char; \
1478 CODING_SPEC_ISO_DESIGNATION (coding, reg) = charset; \
1479 } while (0)
1481 /* The following two macros produce codes (control character or escape
1482 sequence) for ISO2022 single-shift functions (single-shift-2 and
1483 single-shift-3). */
1485 #define ENCODE_SINGLE_SHIFT_2 \
1486 do { \
1487 if (coding->flags & CODING_FLAG_ISO_SEVEN_BITS) \
1489 else \
1490 { \
1491 *dst++ = ISO_CODE_SS2; \
1492 coding->fake_multibyte = 1; \
1493 } \
1494 CODING_SPEC_ISO_SINGLE_SHIFTING (coding) = 1; \
1495 } while (0)
1497 #define ENCODE_SINGLE_SHIFT_3 \
1498 do { \
1499 if (coding->flags & CODING_FLAG_ISO_SEVEN_BITS) \
1501 else \
1502 { \
1503 *dst++ = ISO_CODE_SS3; \
1504 coding->fake_multibyte = 1; \
1505 } \
1506 CODING_SPEC_ISO_SINGLE_SHIFTING (coding) = 1; \
1507 } while (0)
1509 /* The following four macros produce codes (control character or
1510 escape sequence) for ISO2022 locking-shift functions (shift-in,
1511 shift-out, locking-shift-2, and locking-shift-3). */
1513 #define ENCODE_SHIFT_IN \
1514 do { \
1515 *dst++ = ISO_CODE_SI; \
1516 CODING_SPEC_ISO_INVOCATION (coding, 0) = 0; \
1517 } while (0)
1519 #define ENCODE_SHIFT_OUT \
1520 do { \
1521 *dst++ = ISO_CODE_SO; \
1522 CODING_SPEC_ISO_INVOCATION (coding, 0) = 1; \
1523 } while (0)
1525 #define ENCODE_LOCKING_SHIFT_2 \
1526 do { \
1528 CODING_SPEC_ISO_INVOCATION (coding, 0) = 2; \
1529 } while (0)
1531 #define ENCODE_LOCKING_SHIFT_3 \
1532 do { \
1534 CODING_SPEC_ISO_INVOCATION (coding, 0) = 3; \
1535 } while (0)
1537 /* Produce codes for a DIMENSION1 character whose character set is
1538 CHARSET and whose position-code is C1. Designation and invocation
1539 sequences are also produced in advance if necessary. */
1542 #define ENCODE_ISO_CHARACTER_DIMENSION1(charset, c1) \
1543 do { \
1544 if (CODING_SPEC_ISO_SINGLE_SHIFTING (coding)) \
1545 { \
1546 if (coding->flags & CODING_FLAG_ISO_SEVEN_BITS) \
1547 *dst++ = c1 & 0x7F; \
1548 else \
1549 *dst++ = c1 | 0x80; \
1550 CODING_SPEC_ISO_SINGLE_SHIFTING (coding) = 0; \
1551 break; \
1552 } \
1553 else if (charset == CODING_SPEC_ISO_PLANE_CHARSET (coding, 0)) \
1554 { \
1555 *dst++ = c1 & 0x7F; \
1556 break; \
1557 } \
1558 else if (charset == CODING_SPEC_ISO_PLANE_CHARSET (coding, 1)) \
1559 { \
1560 *dst++ = c1 | 0x80; \
1561 break; \
1562 } \
1563 else if (coding->flags & CODING_FLAG_ISO_SAFE \
1564 && !coding->safe_charsets[charset]) \
1565 { \
1566 /* We should not encode this character, instead produce one or \
1568 *dst++ = CODING_INHIBIT_CHARACTER_SUBSTITUTION; \
1569 if (CHARSET_WIDTH (charset) == 2) \
1570 *dst++ = CODING_INHIBIT_CHARACTER_SUBSTITUTION; \
1571 break; \
1572 } \
1573 else \
1574 /* Since CHARSET is not yet invoked to any graphic planes, we \
1575 must invoke it, or, at first, designate it to some graphic \
1576 register. Then repeat the loop to actually produce the \
1578 dst = encode_invocation_designation (charset, coding, dst); \
1579 } while (1)
1581 /* Produce codes for a DIMENSION2 character whose character set is
1582 CHARSET and whose position-codes are C1 and C2. Designation and
1583 invocation codes are also produced in advance if necessary. */
1585 #define ENCODE_ISO_CHARACTER_DIMENSION2(charset, c1, c2) \
1586 do { \
1587 if (CODING_SPEC_ISO_SINGLE_SHIFTING (coding)) \
1588 { \
1589 if (coding->flags & CODING_FLAG_ISO_SEVEN_BITS) \
1590 *dst++ = c1 & 0x7F, *dst++ = c2 & 0x7F; \
1591 else \
1592 *dst++ = c1 | 0x80, *dst++ = c2 | 0x80; \
1593 CODING_SPEC_ISO_SINGLE_SHIFTING (coding) = 0; \
1594 break; \
1595 } \
1596 else if (charset == CODING_SPEC_ISO_PLANE_CHARSET (coding, 0)) \
1597 { \
1598 *dst++ = c1 & 0x7F, *dst++= c2 & 0x7F; \
1599 break; \
1600 } \
1601 else if (charset == CODING_SPEC_ISO_PLANE_CHARSET (coding, 1)) \
1602 { \
1603 *dst++ = c1 | 0x80, *dst++= c2 | 0x80; \
1604 break; \
1605 } \
1606 else if (coding->flags & CODING_FLAG_ISO_SAFE \
1607 && !coding->safe_charsets[charset]) \
1608 { \
1609 /* We should not encode this character, instead produce one or \
1611 *dst++ = CODING_INHIBIT_CHARACTER_SUBSTITUTION; \
1612 if (CHARSET_WIDTH (charset) == 2) \
1613 *dst++ = CODING_INHIBIT_CHARACTER_SUBSTITUTION; \
1614 break; \
1615 } \
1616 else \
1617 /* Since CHARSET is not yet invoked to any graphic planes, we \
1618 must invoke it, or, at first, designate it to some graphic \
1619 register. Then repeat the loop to actually produce the \
1621 dst = encode_invocation_designation (charset, coding, dst); \
1622 } while (1)
1624 #define ENCODE_ISO_CHARACTER(charset, c1, c2) \
1625 do { \
1626 int c_alt, charset_alt; \
1627 if (!NILP (translation_table) \
1628 && ((c_alt = translate_char (translation_table, -1, \
1629 charset, c1, c2)) \
1630 >= 0)) \
1631 SPLIT_CHAR (c_alt, charset_alt, c1, c2); \
1632 else \
1633 charset_alt = charset; \
1634 if (CHARSET_DEFINED_P (charset_alt)) \
1635 { \
1636 if (CHARSET_DIMENSION (charset_alt) == 1) \
1637 { \
1638 if (charset == CHARSET_ASCII \
1639 && coding->flags & CODING_FLAG_ISO_USE_ROMAN) \
1640 charset_alt = charset_latin_jisx0201; \
1641 ENCODE_ISO_CHARACTER_DIMENSION1 (charset_alt, c1); \
1642 } \
1643 else \
1644 { \
1645 if (charset == charset_jisx0208 \
1646 && coding->flags & CODING_FLAG_ISO_USE_OLDJIS) \
1647 charset_alt = charset_jisx0208_1978; \
1648 ENCODE_ISO_CHARACTER_DIMENSION2 (charset_alt, c1, c2); \
1649 } \
1650 } \
1651 else \
1652 { \
1653 if (coding->flags & CODING_FLAG_ISO_SEVEN_BITS) \
1654 { \
1655 *dst++ = charset & 0x7f; \
1656 *dst++ = c1 & 0x7f; \
1657 if (c2) \
1658 *dst++ = c2 & 0x7f; \
1659 } \
1660 else \
1661 { \
1662 *dst++ = charset; \
1663 *dst++ = c1; \
1664 if (c2) \
1665 *dst++ = c2; \
1666 } \
1667 } \
1668 if (! COMPOSING_P (coding->composing)) \
1669 coding->consumed_char++; \
1670 } while (0)
1672 /* Produce designation and invocation codes at a place pointed by DST
1673 to use CHARSET. The element `spec.iso2022' of *CODING is updated.
1674 Return new DST. */
1681 {
1684 /* At first, check designations. */
1690 {
1691 /* CHARSET is not yet designated to any graphic registers. */
1692 /* At first check the requested designation. */
1695 /* Since CHARSET requests no special designation, designate it
1696 to graphic register 0. */
1700 }
1704 {
1705 /* Since the graphic register REG is not invoked to any graphic
1706 planes, invoke it to graphic plane 0. */
1708 {
1710 ENCODE_SHIFT_IN;
1714 ENCODE_SHIFT_OUT;
1719 ENCODE_SINGLE_SHIFT_2;
1720 else
1721 ENCODE_LOCKING_SHIFT_2;
1726 ENCODE_SINGLE_SHIFT_3;
1727 else
1728 ENCODE_LOCKING_SHIFT_3;
1730 }
1731 }
1733 }
1735 /* The following two macros produce codes for indicating composition. */
1740 /* The following three macros produce codes for indicating direction
1741 of text. */
1742 #define ENCODE_CONTROL_SEQUENCE_INTRODUCER \
1743 do { \
1744 if (coding->flags == CODING_FLAG_ISO_SEVEN_BITS) \
1746 else \
1747 *dst++ = ISO_CODE_CSI; \
1748 } while (0)
1750 #define ENCODE_DIRECTION_R2L \
1753 #define ENCODE_DIRECTION_L2R \
1756 /* Produce codes for designation and invocation to reset the graphic
1757 planes and registers to initial state. */
1758 #define ENCODE_RESET_PLANE_AND_REGISTER \
1759 do { \
1760 int reg; \
1761 if (CODING_SPEC_ISO_INVOCATION (coding, 0) != 0) \
1762 ENCODE_SHIFT_IN; \
1763 for (reg = 0; reg < 4; reg++) \
1764 if (CODING_SPEC_ISO_INITIAL_DESIGNATION (coding, reg) >= 0 \
1765 && (CODING_SPEC_ISO_DESIGNATION (coding, reg) \
1766 != CODING_SPEC_ISO_INITIAL_DESIGNATION (coding, reg))) \
1767 ENCODE_DESIGNATION \
1768 (CODING_SPEC_ISO_INITIAL_DESIGNATION (coding, reg), reg, coding); \
1769 } while (0)
1771 /* Produce designation sequences of charsets in the line started from
1772 SRC to a place pointed by *DSTP, and update DSTP.
1774 If the current block ends before any end-of-line, we may fail to
1775 find all the necessary designations. */
1777 void
1780 Lisp_Object table;
1782 {
1784 /* Table of charsets to be designated to each graphic register. */
1792 {
1797 else
1798 {
1805 }
1809 {
1810 found++;
1812 }
1815 }
1818 {
1824 }
1825 }
1827 /* See the above "GENERAL NOTES on `encode_coding_XXX ()' functions". */
1829 int
1834 {
1839 /* Since the maximum bytes produced by each loop is 20, we subtract 19
1840 from DST_END to assure overflow checking is necessary only at the
1841 head of loop. */
1843 Lisp_Object translation_table
1855 {
1856 /* SRC_BASE remembers the start position in source in each loop.
1857 The loop will be exited when there's not enough source text
1858 to analyze multi-byte codes (within macros ONE_MORE_BYTE,
1859 TWO_MORE_BYTES, and THREE_MORE_BYTES). In that case, SRC is
1860 reset to SRC_BASE before exiting. */
1866 {
1867 /* We have to produce designation sequences if any now. */
1871 }
1874 /* If we are seeing a component of a composite character, we are
1875 seeing a leading-code encoded irregularly for composition, or
1876 a composition rule if composing with rule. We must set C1 to
1877 a normal leading-code or an ASCII code. If we are not seeing
1878 a composite character, we must reset composition,
1879 designation, and invocation states. */
1881 {
1883 {
1884 /* We are not in a composite character any longer. */
1886 ENCODE_RESET_PLANE_AND_REGISTER;
1887 ENCODE_COMPOSITION_END;
1888 }
1889 else
1890 {
1892 {
1896 }
1900 {
1901 /* This is an ASCII component. */
1904 }
1905 else
1906 /* This is a leading-code of non ASCII component. */
1908 }
1909 }
1911 /* Now encode one character. C1 is a control character, an
1912 ASCII character, or a leading-code of multi-byte character. */
1914 {
1922 ENCODE_RESET_PLANE_AND_REGISTER;
1929 {
1931 ENCODE_RESET_PLANE_AND_REGISTER;
1935 }
1936 /* fall down to treat '\r' as '\n' ... */
1940 ENCODE_RESET_PLANE_AND_REGISTER;
1950 else
1960 {
1961 /* invalid sequence */
1963 src--;
1965 }
1966 else
1974 {
1975 /* invalid sequence */
1979 }
1982 else
1989 {
1990 /* invalid sequence */
1994 }
1995 else
2002 {
2003 /* invalid sequence */
2005 src--;
2007 }
2009 {
2010 ENCODE_RESET_PLANE_AND_REGISTER;
2012 ENCODE_COMPOSITION_WITH_RULE_START;
2014 }
2015 else
2016 {
2017 ENCODE_RESET_PLANE_AND_REGISTER;
2018 /* Rewind one byte because it is a character code of
2019 composition elements. */
2020 src--;
2022 ENCODE_COMPOSITION_NO_RULE_START;
2024 }
2029 ENCODE_RESET_PLANE_AND_REGISTER;
2033 }
2035 label_end_of_loop:
2039 }
2044 /* If this is the last block of the text to be encoded, we must
2045 reset graphic planes and registers to the initial state, and
2046 flush out the carryover if any. */
2048 {
2049 ENCODE_RESET_PLANE_AND_REGISTER;
2051 ENCODE_COMPOSITION_END;
2053 {
2056 }
2057 }
2061 }
2063 \f
2064 /*** 4. SJIS and BIG5 handlers ***/
2066 /* Although SJIS and BIG5 are not ISO's coding system, they are used
2067 quite widely. So, for the moment, Emacs supports them in the bare
2068 C code. But, in the future, they may be supported only by CCL. */
2070 /* SJIS is a coding system encoding three character sets: ASCII, right
2071 half of JISX0201-Kana, and JISX0208. An ASCII character is encoded
2072 as is. A character of charset katakana-jisx0201 is encoded by
2073 "position-code + 0x80". A character of charset japanese-jisx0208
2074 is encoded in 2-byte but two position-codes are divided and shifted
2075 so that it fit in the range below.
2077 --- CODE RANGE of SJIS ---
2078 (character set) (range)
2079 ASCII 0x00 .. 0x7F
2080 KATAKANA-JISX0201 0xA0 .. 0xDF
2081 JISX0208 (1st byte) 0x81 .. 0x9F and 0xE0 .. 0xEF
2082 (2nd byte) 0x40 .. 0x7E and 0x80 .. 0xFC
2083 -------------------------------
2085 */
2087 /* BIG5 is a coding system encoding two character sets: ASCII and
2088 Big5. An ASCII character is encoded as is. Big5 is a two-byte
2089 character set and is encoded in two-byte.
2091 --- CODE RANGE of BIG5 ---
2092 (character set) (range)
2093 ASCII 0x00 .. 0x7F
2094 Big5 (1st byte) 0xA1 .. 0xFE
2095 (2nd byte) 0x40 .. 0x7E and 0xA1 .. 0xFE
2096 --------------------------
2098 Since the number of characters in Big5 is larger than maximum
2099 characters in Emacs' charset (96x96), it can't be handled as one
2100 charset. So, in Emacs, Big5 is divided into two: `charset-big5-1'
2101 and `charset-big5-2'. Both are DIMENSION2 and CHARS94. The former
2102 contains frequently used characters and the latter contains less
2103 frequently used characters. */
2105 /* Macros to decode or encode a character of Big5 in BIG5. B1 and B2
2106 are the 1st and 2nd position-codes of Big5 in BIG5 coding system.
2107 C1 and C2 are the 1st and 2nd position-codes of of Emacs' internal
2108 format. CHARSET is `charset_big5_1' or `charset_big5_2'. */
2110 /* Number of Big5 characters which have the same code in 1st byte. */
2111 #define BIG5_SAME_ROW (0xFF - 0xA1 + 0x7F - 0x40)
2113 #define DECODE_BIG5(b1, b2, charset, c1, c2) \
2114 do { \
2115 unsigned int temp \
2116 = (b1 - 0xA1) * BIG5_SAME_ROW + b2 - (b2 < 0x7F ? 0x40 : 0x62); \
2117 if (b1 < 0xC9) \
2118 charset = charset_big5_1; \
2119 else \
2120 { \
2121 charset = charset_big5_2; \
2122 temp -= (0xC9 - 0xA1) * BIG5_SAME_ROW; \
2123 } \
2124 c1 = temp / (0xFF - 0xA1) + 0x21; \
2125 c2 = temp % (0xFF - 0xA1) + 0x21; \
2126 } while (0)
2128 #define ENCODE_BIG5(charset, c1, c2, b1, b2) \
2129 do { \
2130 unsigned int temp = (c1 - 0x21) * (0xFF - 0xA1) + (c2 - 0x21); \
2131 if (charset == charset_big5_2) \
2132 temp += BIG5_SAME_ROW * (0xC9 - 0xA1); \
2133 b1 = temp / BIG5_SAME_ROW + 0xA1; \
2134 b2 = temp % BIG5_SAME_ROW; \
2135 b2 += b2 < 0x3F ? 0x40 : 0x62; \
2136 } while (0)
2138 #define DECODE_SJIS_BIG5_CHARACTER(charset, c1, c2) \
2139 do { \
2140 int c_alt, charset_alt = (charset); \
2141 if (!NILP (translation_table) \
2142 && ((c_alt = translate_char (translation_table, \
2143 -1, (charset), c1, c2)) >= 0)) \
2144 SPLIT_CHAR (c_alt, charset_alt, c1, c2); \
2145 if (charset_alt == CHARSET_ASCII || charset_alt < 0) \
2146 DECODE_CHARACTER_ASCII (c1); \
2147 else if (CHARSET_DIMENSION (charset_alt) == 1) \
2148 DECODE_CHARACTER_DIMENSION1 (charset_alt, c1); \
2149 else \
2150 DECODE_CHARACTER_DIMENSION2 (charset_alt, c1, c2); \
2151 } while (0)
2153 #define ENCODE_SJIS_BIG5_CHARACTER(charset, c1, c2) \
2154 do { \
2155 int c_alt, charset_alt; \
2156 if (!NILP (translation_table) \
2157 && ((c_alt = translate_char (translation_table, -1, \
2158 charset, c1, c2)) \
2159 >= 0)) \
2160 SPLIT_CHAR (c_alt, charset_alt, c1, c2); \
2161 else \
2162 charset_alt = charset; \
2163 if (charset_alt == charset_ascii) \
2164 *dst++ = c1; \
2165 else if (CHARSET_DIMENSION (charset_alt) == 1) \
2166 { \
2167 if (sjis_p && charset_alt == charset_katakana_jisx0201) \
2168 *dst++ = c1; \
2169 else if (sjis_p && charset_alt == charset_latin_jisx0201) \
2170 *dst++ = c1 & 0x7F; \
2171 else \
2172 { \
2173 *dst++ = charset_alt, *dst++ = c1; \
2174 coding->fake_multibyte = 1; \
2175 } \
2176 } \
2177 else \
2178 { \
2179 c1 &= 0x7F, c2 &= 0x7F; \
2180 if (sjis_p && (charset_alt == charset_jisx0208 \
2181 || charset_alt == charset_jisx0208_1978))\
2182 { \
2183 unsigned char s1, s2; \
2184 \
2185 ENCODE_SJIS (c1, c2, s1, s2); \
2186 *dst++ = s1, *dst++ = s2; \
2187 coding->fake_multibyte = 1; \
2188 } \
2189 else if (!sjis_p \
2190 && (charset_alt == charset_big5_1 \
2191 || charset_alt == charset_big5_2)) \
2192 { \
2193 unsigned char b1, b2; \
2194 \
2195 ENCODE_BIG5 (charset_alt, c1, c2, b1, b2); \
2196 *dst++ = b1, *dst++ = b2; \
2197 } \
2198 else \
2199 { \
2200 *dst++ = charset_alt, *dst++ = c1, *dst++ = c2; \
2201 coding->fake_multibyte = 1; \
2202 } \
2203 } \
2204 coding->consumed_char++; \
2205 } while (0);
2207 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
2208 Check if a text is encoded in SJIS. If it is, return
2209 CODING_CATEGORY_MASK_SJIS, else return 0. */
2211 int
2214 {
2218 {
2221 {
2224 }
2225 }
2227 }
2229 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
2230 Check if a text is encoded in BIG5. If it is, return
2231 CODING_CATEGORY_MASK_BIG5, else return 0. */
2233 int
2236 {
2240 {
2243 {
2249 }
2250 }
2252 }
2254 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions".
2255 If SJIS_P is 1, decode SJIS text, else decode BIG5 test. */
2257 int
2264 {
2269 /* Since the maximum bytes produced by each loop is 4, we subtract 3
2270 from DST_END to assure overflow checking is necessary only at the
2271 head of loop. */
2273 Lisp_Object translation_table
2285 {
2286 /* SRC_BASE remembers the start position in source in each loop.
2287 The loop will be exited when there's not enough source text
2288 to analyze two-byte character (within macro ONE_MORE_BYTE).
2289 In that case, SRC is reset to SRC_BASE before exiting. */
2294 {
2296 {
2298 {
2303 {
2306 }
2307 else
2308 /* To process C2 again, SRC is subtracted by 1. */
2310 }
2313 else
2315 }
2320 {
2323 }
2324 else
2327 }
2329 {
2332 }
2333 else
2334 {
2336 {
2338 {
2339 /* SJIS -> JISX0208 */
2342 {
2345 }
2346 else
2348 }
2350 /* SJIS -> JISX0201-Kana */
2351 {
2355 }
2356 else
2358 }
2359 else
2360 {
2361 /* BIG5 -> Big5 */
2363 {
2366 {
2371 }
2372 else
2374 }
2375 else
2377 }
2378 }
2381 label_invalid_code_1:
2387 label_invalid_code_2:
2393 label_end_of_loop:
2395 label_end_of_loop_2:
2398 }
2401 {
2406 {
2414 }
2415 }
2420 }
2422 /* See the above "GENERAL NOTES on `encode_coding_XXX ()' functions".
2423 This function can encode `charset_ascii', `charset_katakana_jisx0201',
2424 `charset_jisx0208', `charset_big5_1', and `charset_big5-2'. We are
2425 sure that all these charsets are registered as official charset
2426 (i.e. do not have extended leading-codes). Characters of other
2427 charsets are produced without any encoding. If SJIS_P is 1, encode
2428 SJIS text, else encode BIG5 text. */
2430 int
2437 {
2442 /* Since the maximum bytes produced by each loop is 2, we subtract 1
2443 from DST_END to assure overflow checking is necessary only at the
2444 head of loop. */
2446 Lisp_Object translation_table
2458 {
2459 /* SRC_BASE remembers the start position in source in each loop.
2460 The loop will be exited when there's not enough source text
2461 to analyze multi-byte codes (within macros ONE_MORE_BYTE and
2462 TWO_MORE_BYTES). In that case, SRC is reset to SRC_BASE
2463 before exiting. */
2468 {
2470 {
2473 }
2476 else
2478 }
2481 {
2493 {
2497 }
2498 /* fall down to treat '\r' as '\n' ... */
2506 else
2533 }
2536 label_end_of_loop:
2540 }
2548 }
2550 \f
2551 /*** 5. CCL handlers ***/
2553 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
2554 Check if a text is encoded in a coding system of which
2555 encoder/decoder are written in CCL program. If it is, return
2556 CODING_CATEGORY_MASK_CCL, else return 0. */
2558 int
2561 {
2564 /* No coding system is assigned to coding-category-ccl. */
2570 {
2572 src++;
2573 }
2575 }
2577 \f
2578 /*** 6. End-of-line handlers ***/
2580 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions".
2581 This function is called only when `coding->eol_type' is
2582 CODING_EOL_CRLF or CODING_EOL_CR. */
2584 int
2589 {
2600 {
2604 }
2607 {
2609 {
2610 /* Since the maximum bytes produced by each loop is 2, we
2611 subtract 1 from DST_END to assure overflow checking is
2612 necessary only at the head of loop. */
2618 {
2623 {
2627 else
2628 {
2630 {
2633 }
2634 src--;
2638 }
2639 }
2642 {
2645 }
2646 else
2647 {
2651 }
2654 label_end_of_loop:
2656 label_end_of_loop_2:
2659 }
2661 {
2666 {
2667 /* This is the last block of the text to be decoded.
2668 We flush out all remaining codes. */
2675 }
2676 }
2677 }
2682 {
2684 {
2689 }
2691 {
2694 }
2695 }
2697 {
2700 }
2703 else
2711 {
2714 }
2717 else
2723 }
2728 }
2730 /* See "GENERAL NOTES about `encode_coding_XXX ()' functions". Encode
2731 format of end-of-line according to `coding->eol_type'. If
2732 `coding->mode & CODING_MODE_SELECTIVE_DISPLAY' is nonzero, code
2733 '\r' in source text also means end-of-line. */
2735 int
2740 {
2748 {
2752 /* Since the maximum bytes produced by each loop is 2, we
2753 subtract 1 from DST_END to assure overflow checking is
2754 necessary only at the head of loop. */
2760 {
2765 else
2766 {
2770 }
2771 }
2774 }
2775 else
2776 {
2780 {
2783 }
2786 else
2790 {
2792 {
2797 }
2798 }
2799 else
2800 {
2802 {
2805 }
2807 }
2810 }
2815 }
2817 \f
2818 /*** 7. C library functions ***/
2820 /* In Emacs Lisp, coding system is represented by a Lisp symbol which
2821 has a property `coding-system'. The value of this property is a
2822 vector of length 5 (called as coding-vector). Among elements of
2823 this vector, the first (element[0]) and the fifth (element[4])
2824 carry important information for decoding/encoding. Before
2825 decoding/encoding, this information should be set in fields of a
2826 structure of type `coding_system'.
2828 A value of property `coding-system' can be a symbol of another
2829 subsidiary coding-system. In that case, Emacs gets coding-vector
2830 from that symbol.
2832 `element[0]' contains information to be set in `coding->type'. The
2833 value and its meaning is as follows:
2835 0 -- coding_type_emacs_mule
2836 1 -- coding_type_sjis
2837 2 -- coding_type_iso2022
2838 3 -- coding_type_big5
2839 4 -- coding_type_ccl encoder/decoder written in CCL
2840 nil -- coding_type_no_conversion
2841 t -- coding_type_undecided (automatic conversion on decoding,
2842 no-conversion on encoding)
2844 `element[4]' contains information to be set in `coding->flags' and
2845 `coding->spec'. The meaning varies by `coding->type'.
2847 If `coding->type' is `coding_type_iso2022', element[4] is a vector
2848 of length 32 (of which the first 13 sub-elements are used now).
2849 Meanings of these sub-elements are:
2851 sub-element[N] where N is 0 through 3: to be set in `coding->spec.iso2022'
2852 If the value is an integer of valid charset, the charset is
2853 assumed to be designated to graphic register N initially.
2855 If the value is minus, it is a minus value of charset which
2856 reserves graphic register N, which means that the charset is
2857 not designated initially but should be designated to graphic
2858 register N just before encoding a character in that charset.
2860 If the value is nil, graphic register N is never used on
2861 encoding.
2863 sub-element[N] where N is 4 through 11: to be set in `coding->flags'
2864 Each value takes t or nil. See the section ISO2022 of
2865 `coding.h' for more information.
2867 If `coding->type' is `coding_type_big5', element[4] is t to denote
2868 BIG5-ETen or nil to denote BIG5-HKU.
2870 If `coding->type' takes the other value, element[4] is ignored.
2872 Emacs Lisp's coding system also carries information about format of
2873 end-of-line in a value of property `eol-type'. If the value is
2874 integer, 0 means CODING_EOL_LF, 1 means CODING_EOL_CRLF, and 2
2875 means CODING_EOL_CR. If it is not integer, it should be a vector
2876 of subsidiary coding systems of which property `eol-type' has one
2877 of above values.
2879 */
2881 /* Extract information for decoding/encoding from CODING_SYSTEM_SYMBOL
2882 and set it in CODING. If CODING_SYSTEM_SYMBOL is invalid, CODING
2883 is setup so that no conversion is necessary and return -1, else
2884 return 0. */
2886 int
2888 Lisp_Object coding_system;
2890 {
2892 Lisp_Object val;
2895 /* Initialize some fields required for all kinds of coding systems. */
2914 {
2917 }
2919 {
2921 coding->common_flags
2923 }
2925 {
2927 coding->common_flags
2929 }
2930 else
2934 /* Try short cut. */
2936 {
2938 {
2941 }
2942 else
2945 }
2947 /* Initialize remaining fields. */
2951 /* Get values of coding system properties:
2952 `post-read-conversion', `pre-write-conversion',
2953 `translation-table-for-decode', `translation-table-for-encode'. */
2955 /* Pre & post conversion functions should be disabled if
2956 inhibit_eol_conversion is nozero. This is the case that a code
2957 conversion function is called while those functions are running. */
2959 {
2962 }
2973 {
2977 else
2979 }
2980 else
2985 {
2988 }
2989 else
2990 {
2993 {
2997 }
2998 }
3001 {
3012 coding->common_flags
3018 coding->common_flags
3020 {
3031 coding->flags
3045 );
3047 /* Invoke graphic register 0 to plane 0. */
3049 /* Invoke graphic register 1 to plane 1 if we can use full 8-bit. */
3052 /* Not single shifting at first. */
3054 /* Beginning of buffer should also be regarded as bol. */
3061 {
3068 }
3070 /* Checks FLAGS[REG] (REG = 0, 1, 2 3) and decide designations.
3071 FLAGS[REG] can be one of below:
3072 integer CHARSET: CHARSET occupies register I,
3073 t: designate nothing to REG initially, but can be used
3074 by any charsets,
3075 list of integer, nil, or t: designate the first
3076 element (if integer) to REG initially, the remaining
3077 elements (if integer) is designated to REG on request,
3078 if an element is t, REG can be used by any charsets,
3079 nil: REG is never used. */
3084 {
3088 {
3091 }
3093 {
3097 }
3099 {
3100 Lisp_Object tail;
3108 {
3111 }
3112 else
3116 {
3126 }
3127 }
3128 else
3133 }
3136 {
3137 /* REG 1 can be used only by locking shift in 7-bit env. */
3141 /* Without any shifting, only REG 0 and 1 can be used. */
3143 }
3147 {
3149 {
3150 /* There exist some default graphic registers to be
3151 used CHARSET. */
3153 /* We had better avoid designating a charset of
3154 CHARS96 to REG 0 as far as possible. */
3159 else
3163 }
3164 }
3165 }
3172 coding->common_flags
3174 coding->flags
3176 ? CODING_FLAG_BIG5_HKU
3182 coding->common_flags
3184 {
3196 {
3200 {
3208 {
3215 }
3216 }
3217 }
3218 }
3228 }
3231 label_invalid_coding_system:
3238 }
3240 /* Setup raw-text or one of its subsidiaries in the structure
3241 coding_system CODING according to the already setup value eol_type
3242 in CODING. CODING should be setup for some coding system in
3243 advance. */
3245 void
3248 {
3250 {
3254 {
3255 Lisp_Object subsidiaries;
3260 coding->symbol
3262 }
3264 }
3266 }
3268 /* Emacs has a mechanism to automatically detect a coding system if it
3269 is one of Emacs' internal format, ISO2022, SJIS, and BIG5. But,
3270 it's impossible to distinguish some coding systems accurately
3271 because they use the same range of codes. So, at first, coding
3272 systems are categorized into 7, those are:
3274 o coding-category-emacs-mule
3276 The category for a coding system which has the same code range
3277 as Emacs' internal format. Assigned the coding-system (Lisp
3278 symbol) `emacs-mule' by default.
3280 o coding-category-sjis
3282 The category for a coding system which has the same code range
3283 as SJIS. Assigned the coding-system (Lisp
3284 symbol) `japanese-shift-jis' by default.
3286 o coding-category-iso-7
3288 The category for a coding system which has the same code range
3289 as ISO2022 of 7-bit environment. This doesn't use any locking
3290 shift and single shift functions. This can encode/decode all
3291 charsets. Assigned the coding-system (Lisp symbol)
3292 `iso-2022-7bit' by default.
3294 o coding-category-iso-7-tight
3296 Same as coding-category-iso-7 except that this can
3297 encode/decode only the specified charsets.
3299 o coding-category-iso-8-1
3301 The category for a coding system which has the same code range
3302 as ISO2022 of 8-bit environment and graphic plane 1 used only
3303 for DIMENSION1 charset. This doesn't use any locking shift
3304 and single shift functions. Assigned the coding-system (Lisp
3305 symbol) `iso-latin-1' by default.
3307 o coding-category-iso-8-2
3309 The category for a coding system which has the same code range
3310 as ISO2022 of 8-bit environment and graphic plane 1 used only
3311 for DIMENSION2 charset. This doesn't use any locking shift
3312 and single shift functions. Assigned the coding-system (Lisp
3313 symbol) `japanese-iso-8bit' by default.
3315 o coding-category-iso-7-else
3317 The category for a coding system which has the same code range
3318 as ISO2022 of 7-bit environemnt but uses locking shift or
3319 single shift functions. Assigned the coding-system (Lisp
3320 symbol) `iso-2022-7bit-lock' by default.
3322 o coding-category-iso-8-else
3324 The category for a coding system which has the same code range
3325 as ISO2022 of 8-bit environemnt but uses locking shift or
3326 single shift functions. Assigned the coding-system (Lisp
3327 symbol) `iso-2022-8bit-ss2' by default.
3329 o coding-category-big5
3331 The category for a coding system which has the same code range
3332 as BIG5. Assigned the coding-system (Lisp symbol)
3333 `cn-big5' by default.
3335 o coding-category-ccl
3337 The category for a coding system of which encoder/decoder is
3338 written in CCL programs. The default value is nil, i.e., no
3339 coding system is assigned.
3341 o coding-category-binary
3343 The category for a coding system not categorized in any of the
3344 above. Assigned the coding-system (Lisp symbol)
3345 `no-conversion' by default.
3347 Each of them is a Lisp symbol and the value is an actual
3348 `coding-system's (this is also a Lisp symbol) assigned by a user.
3349 What Emacs does actually is to detect a category of coding system.
3350 Then, it uses a `coding-system' assigned to it. If Emacs can't
3351 decide only one possible category, it selects a category of the
3352 highest priority. Priorities of categories are also specified by a
3353 user in a Lisp variable `coding-category-list'.
3355 */
3357 static
3360 /* Detect how a text of length SRC_BYTES pointed by SOURCE is encoded.
3361 If it detects possible coding systems, return an integer in which
3362 appropriate flag bits are set. Flag bits are defined by macros
3363 CODING_CATEGORY_MASK_XXX in `coding.h'.
3365 How many ASCII characters are at the head is returned as *SKIP. */
3367 static int
3371 {
3377 /* At first, skip all ASCII characters and control characters except
3378 for three ISO2022 specific control characters. */
3383 label_loop_detect_coding:
3388 /* We found nothing other than ASCII. There's nothing to do. */
3392 /* The text seems to be encoded in some multilingual coding system.
3393 Now, try to find in which coding system the text is encoded. */
3395 {
3396 /* i.e. (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO) */
3397 /* C is an ISO2022 specific control code of C0. */
3400 {
3401 /* No valid ISO2022 code follows C. Try again. */
3402 src++;
3405 else
3408 }
3411 }
3412 else
3413 {
3417 {
3418 /* C is the first byte of SJIS character code,
3419 or a leading-code of Emacs' internal format (emacs-mule). */
3422 /* Or, if C is a special latin extra code,
3423 or is an ISO2022 specific control code of C1 (SS2 or SS3),
3424 or is an ISO2022 control-sequence-introducer (CSI),
3425 we should also consider the possibility of ISO2022 codings. */
3437 }
3438 else
3439 /* C is a character of ISO2022 in graphic plane right,
3440 or a SJIS's 1-byte character code (i.e. JISX0201),
3441 or the first byte of BIG5's 2-byte code. */
3443 | CODING_CATEGORY_MASK_ISO_8BIT
3444 | CODING_CATEGORY_MASK_SJIS
3447 /* Or, we may have to consider the possibility of CCL. */
3455 {
3457 {
3474 }
3476 }
3487 }
3490 label_return_highest_only:
3492 {
3495 }
3497 }
3499 /* Detect how a text of length SRC_BYTES pointed by SRC is encoded.
3500 The information of the detected coding system is set in CODING. */
3502 void
3507 {
3510 Lisp_Object val;
3518 /* We found a single coding system of the highest priority in MASK. */
3527 {
3528 Lisp_Object tmp;
3533 }
3535 /* Set this again because setup_coding_system reset this member. */
3537 }
3539 /* Detect how end-of-line of a text of length SRC_BYTES pointed by
3540 SOURCE is encoded. Return one of CODING_EOL_LF, CODING_EOL_CRLF,
3541 CODING_EOL_CR, and CODING_EOL_UNDECIDED.
3543 How many non-eol characters are at the head is returned as *SKIP. */
3545 #define MAX_EOL_CHECK_COUNT 3
3547 static int
3551 {
3561 {
3564 {
3567 total++;
3572 else
3576 /* This is the first end-of-line. */
3579 {
3580 /* The found type is different from what found before. */
3583 }
3584 }
3585 }
3590 }
3592 /* Detect how end-of-line of a text of length SRC_BYTES pointed by SRC
3593 is encoded. If it detects an appropriate format of end-of-line, it
3594 sets the information in *CODING. */
3596 void
3601 {
3602 Lisp_Object val;
3608 else
3614 {
3615 #if 0
3616 /* This code is suppressed until we find a better way to
3617 distinguish raw text file and binary file. */
3619 /* If we have already detected that the coding is raw-text, the
3620 coding should actually be no-conversion. */
3622 {
3625 }
3626 /* Else, let's decode only text code anyway. */
3629 }
3633 {
3636 }
3637 }
3639 #define CONVERSION_BUFFER_EXTRA_ROOM 256
3641 #define DECODING_BUFFER_MAG(coding) \
3642 (coding->type == coding_type_iso2022 \
3643 ? 3 \
3644 : ((coding->type == coding_type_sjis || coding->type == coding_type_big5) \
3645 ? 2 \
3646 : (coding->type == coding_type_raw_text \
3647 ? 1 \
3648 : (coding->type == coding_type_ccl \
3649 ? coding->spec.ccl.decoder.buf_magnification \
3650 : 2))))
3652 /* Return maximum size (bytes) of a buffer enough for decoding
3653 SRC_BYTES of text encoded in CODING. */
3655 int
3659 {
3662 }
3664 /* Return maximum size (bytes) of a buffer enough for encoding
3665 SRC_BYTES of text to CODING. */
3667 int
3671 {
3676 else
3680 }
3682 #ifndef MINIMUM_CONVERSION_BUFFER_SIZE
3683 #define MINIMUM_CONVERSION_BUFFER_SIZE 1024
3684 #endif
3689 /* Return a pointer to a SIZE bytes of buffer to be used for encoding
3690 or decoding. Sufficient memory is allocated automatically. If we
3691 run out of memory, return NULL. */
3696 {
3698 {
3707 }
3709 }
3711 int
3716 {
3725 coding->produced_char
3726 = (encodep
3729 coding->consumed_char
3733 {
3747 }
3749 }
3751 /* See "GENERAL NOTES about `decode_coding_XXX ()' functions". Before
3752 decoding, it may detect coding system and format of end-of-line if
3753 those are not yet decided.
3755 This function does not make full use of DESTINATION buffer. For
3756 instance, if coding->type is coding_type_iso2022, it uses only
3757 (DST_BYTES - 7) bytes of DESTINATION buffer. In the case that
3758 DST_BYTES is decided by the function decoding_buffer_size, it
3759 contains extra 256 bytes (defined by CONVERSION_BUFFER_EXTRA_ROOM).
3760 So, this function can decode the full SOURCE. But, in the other
3761 case, if you want to avoid carry over, you must supply at least 7
3762 bytes more area in DESTINATION buffer than expected maximum bytes
3763 that will be produced by this function. */
3765 int
3770 {
3777 {
3782 }
3791 {
3822 label_no_conversion:
3824 {
3827 }
3828 else
3829 {
3832 }
3835 else
3838 coding->consumed
3841 }
3844 }
3846 /* See "GENERAL NOTES about `encode_coding_XXX ()' functions".
3848 This function does not make full use of DESTINATION buffer. For
3849 instance, if coding->type is coding_type_iso2022, it uses only
3850 (DST_BYTES - 20) bytes of DESTINATION buffer. In the case that
3851 DST_BYTES is decided by the function encoding_buffer_size, it
3852 contains extra 256 bytes (defined by CONVERSION_BUFFER_EXTRA_ROOM).
3853 So, this function can encode the full SOURCE. But, in the other
3854 case, if you want to avoid carry over, you must supply at least 20
3855 bytes more area in DESTINATION buffer than expected maximum bytes
3856 that will be produced by this function. */
3858 int
3863 {
3869 {
3874 }
3877 {
3908 label_no_conversion:
3910 {
3913 }
3914 else
3915 {
3918 }
3921 else
3924 {
3928 }
3930 coding->consumed
3933 }
3936 }
3938 /* Scan text in the region between *BEG and *END (byte positions),
3939 skip characters which we don't have to decode by coding system
3940 CODING at the head and tail, then set *BEG and *END to the region
3941 of the text we actually have to convert. The caller should move
3942 the gap out of the region in advance.
3944 If STR is not NULL, *BEG and *END are indices into STR. */
3946 static void
3951 {
3954 Lisp_Object translation_table;
3959 {
3960 /* We can't skip any data. */
3962 }
3964 {
3965 /* We need no conversion, but don't have to skip any data here.
3966 Decoding routine handles them effectively anyway. */
3968 }
3974 {
3980 /* Some ASCII character should be tranlsated. We give up
3981 shrinking. */
3983 }
3988 /* Detection routine has already found how much we can skip at the
3989 head. */
3993 {
3996 }
3997 else
3998 {
4001 }
4004 {
4008 {
4012 endp--;
4013 /* Do not consider LF as ascii if preceded by CR, since that
4014 confuses eol decoding. */
4016 endp++;
4017 }
4018 else
4024 /* We can skip all ASCII characters at the head. */
4026 {
4029 else
4031 }
4032 /* We can skip all ASCII characters at the tail except for the
4033 second byte of SJIS or BIG5 code. */
4036 else
4038 /* Do not consider LF as ascii if preceded by CR, since that
4039 confuses eol decoding. */
4041 endp++;
4043 endp++;
4048 /* We can't skip any data. */
4051 {
4052 /* We can skip all ASCII characters at the head except for a
4053 few control codes. */
4058 begp++;
4059 }
4061 {
4064 /* We can skip all ASCII characters at the tail. */
4067 else
4069 /* Do not consider LF as ascii if preceded by CR, since that
4070 confuses eol decoding. */
4072 endp++;
4077 {
4078 /* We can skip all charactes at the tail except for 8-bit
4079 codes and ESC and the following 2-byte at the tail. */
4085 {
4087 endp--;
4088 }
4089 else
4092 {
4094 endp--;
4095 }
4096 /* Do not consider LF as ascii if preceded by CR, since that
4097 confuses eol decoding. */
4100 endp++;
4102 {
4104 /* This is an ASCII designation sequence. We can
4105 surely skip the tail. But, if we have
4106 encountered an 8-bit code, skip only the codes
4107 after that. */
4109 else
4110 /* Hmmm, we can't skip the tail. */
4112 }
4115 }
4116 }
4117 }
4121 }
4123 /* Like shrink_decoding_region but for encoding. */
4125 static void
4130 {
4133 Lisp_Object translation_table;
4136 /* We can't skip any data. */
4139 {
4140 /* We need no conversion. */
4143 }
4149 {
4155 /* Some ASCII character should be tranlsated. We give up
4156 shrinking. */
4158 }
4161 {
4164 }
4165 else
4166 {
4169 }
4174 /* Here, we don't have to check coding->pre_write_conversion because
4175 the caller is expected to have handled it already. */
4177 {
4182 {
4185 }
4186 else
4192 /* We can't skip any data. */
4195 {
4198 {
4199 begp++;
4202 }
4205 }
4206 /* fall down ... */
4209 /* We can skip all ASCII characters at the head and tail. */
4212 else
4214 label_skip_tail:
4217 else
4220 }
4225 }
4227 /* As shrinking conversion region requires some overhead, we don't try
4228 shrinking if the length of conversion region is less than this
4229 value. */
4232 #define SHRINK_CONVERSION_REGION(beg, end, coding, str, encodep) \
4233 do { \
4234 if (*(end) - *(beg) > shrink_conversion_region_threshhold) \
4235 { \
4236 if (encodep) shrink_encoding_region (beg, end, coding, str); \
4237 else shrink_decoding_region (beg, end, coding, str); \
4238 } \
4239 } while (0)
4241 static Lisp_Object
4243 Lisp_Object dummy;
4244 {
4247 }
4249 /* Decode (if ENCODEP is zero) or encode (if ENCODEP is nonzero) the
4250 text from FROM to TO (byte positions are FROM_BYTE and TO_BYTE) by
4251 coding system CODING, and return the status code of code conversion
4252 (currently, this value has no meaning).
4254 How many characters (and bytes) are converted to how many
4255 characters (and bytes) are recorded in members of the structure
4256 CODING.
4258 If REPLACE is nonzero, we do various things as if the original text
4259 is deleted and a new text is inserted. See the comments in
4260 replace_range (insdel.c) to know what we are doing. */
4262 int
4266 {
4270 Lisp_Object saved_coding_symbol;
4275 Lisp_Object deletion;
4283 {
4286 }
4289 {
4294 {
4298 else
4301 }
4302 }
4305 {
4306 /* We must detect encoding of text and eol format. */
4311 {
4314 /* It seems that the text contains only ASCII, but we
4315 should not left it undecided because the deeper
4316 decoding routine (decode_coding) tries to detect the
4317 encodings again in vain. */
4319 }
4321 {
4326 /* We had better recover the original eol format if we
4327 encounter an inconsitent eol format while decoding. */
4329 }
4330 }
4337 {
4340 && ! replace
4341 /* See the comment of the member heading_ascii in coding.h. */
4343 {
4344 /* We still may have to combine byte at the head and the
4345 tail of the text in the region. */
4351 }
4352 else
4353 {
4357 }
4359 }
4361 /* Now we convert the text. */
4363 /* For encoding, we must process pre-write-conversion in advance. */
4368 {
4369 /* The function in pre-write-conversion may put a new text in a
4370 new buffer. */
4376 /* We should not call any more pre-write/post-read-conversion
4377 functions while this pre-write-conversion is running. */
4382 /* Discard the unwind protect. */
4383 specpdl_ptr--;
4386 {
4404 }
4405 }
4410 /* Try to skip the heading and tailing ASCIIs. */
4411 {
4421 {
4425 /* We must record and adjust for this new text now. */
4428 }
4436 }
4438 /* The code conversion routine can not preserve text properties for
4439 now. So, we must remove all text properties in the region.
4440 Here, we must suppress all modification hooks. */
4442 {
4447 }
4449 /* For converion, we must put the gap before the text in addition to
4450 making the gap larger for efficient decoding. The required gap
4451 size starts from 2000 which is the magic number used in make_gap.
4452 But, after one batch of conversion, it will be incremented if we
4453 find that it is not enough . */
4475 {
4478 /* The buffer memory is changed from:
4479 +--------+converted-text+---------+-------original-text------+---+
4480 |<-from->|<--inserted-->|---------|<-----------len---------->|---|
4481 |<------------------- GAP_SIZE -------------------->| */
4484 else
4486 /* to:
4487 +--------+-------converted-text--------+--+---original-text--+---+
4488 |<-from->|<--inserted-->|<--produced-->|--|<-(len-consumed)->|---|
4489 |<------------------- GAP_SIZE -------------------->| */
4502 {
4505 }
4507 {
4509 Lisp_Object eol_type;
4511 /* Encode LFs back to the original eol format (CR or CRLF). */
4513 {
4515 }
4516 else
4517 {
4522 {
4523 /* We don't have sufficient room for encoding LFs
4524 back to CRLF. We must record converted and
4525 not-yet-converted text back to the buffer
4526 content, enlarge the gap, then record them out of
4527 the buffer contents again. */
4537 /* Don't forget to update SRC, DST, and PEND. */
4541 }
4547 {
4550 }
4551 }
4553 /* Suppress eol-format conversion in the further conversion. */
4556 /* Set the coding system symbol to that for Unix-like EOL. */
4562 else
4566 }
4568 {
4574 }
4576 {
4577 /* The source text ends in invalid codes. Let's just
4578 make them valid buffer contents, and finish conversion. */
4585 }
4587 {
4588 /* The conversion procedure was interrupted by a user. */
4591 }
4592 /* Now RESULT == CODING_FINISH_INSUFFICIENT_DST */
4594 {
4595 /* It's quite strange to require more memory without
4596 consuming any bytes. Perhaps CCL program bug. */
4599 }
4601 {
4602 /* We have just done the first batch of conversion which was
4603 stoped because of insufficient gap. Let's reconsider the
4604 required gap size (i.e. SRT - DST) now.
4606 We have converted ORIG bytes (== coding->consumed) into
4607 NEW bytes (coding->produced). To convert the remaining
4608 LEN bytes, we may need REQUIRE bytes of gap, where:
4609 REQUIRE + LEN_BYTE = LEN_BYTE * (NEW / ORIG)
4610 REQUIRE = LEN_BYTE * (NEW - ORIG) / ORIG
4611 Here, we are sure that NEW >= ORIG. */
4616 }
4618 {
4619 /* See the comment above the previous call of make_gap. */
4629 /* Don't forget to update SRC, DST. */
4632 }
4633 }
4637 && (encodep
4638 || fake_multibyte
4642 /* If we have shrinked the conversion area, adjust it now. */
4644 {
4654 }
4661 {
4662 Lisp_Object val;
4669 /* We should not call any more pre-write/post-read-conversion
4670 functions while this post-read-conversion is running. */
4674 /* Discard the unwind protect. */
4675 specpdl_ptr--;
4678 }
4681 {
4684 else
4687 }
4691 {
4696 }
4699 }
4701 Lisp_Object
4703 Lisp_Object str;
4706 {
4712 Lisp_Object saved_coding_symbol;
4718 {
4719 /* Since we have to call Lisp functions which assume target text
4720 is in a buffer, after setting a temporary buffer, call
4721 code_convert_region. */
4732 /* We must insert the contents of STR as is without
4733 unibyte<->multibyte conversion. For that, we adjust the
4734 multibyteness of the working buffer to that of STR. */
4738 UNGCPRO;
4740 /* Make a unibyte string if we are encoding, otherwise make a
4741 multibyte string. */
4745 }
4748 {
4749 /* See the comments in code_convert_region. */
4751 {
4755 }
4757 {
4762 /* We had better recover the original eol format if we
4763 encounter an inconsitent eol format while decoding. */
4765 }
4766 }
4772 else
4773 {
4774 /* Try to skip the heading and tailing ASCIIs. */
4776 encodep);
4777 }
4784 else
4789 UNGCPRO;
4793 result = (encodep
4799 {
4800 /* We simple try to decode the whole string again but without
4801 eol-conversion this time. */
4805 }
4813 else
4814 {
4819 }
4822 }
4824 \f
4825 #ifdef emacs
4826 /*** 8. Emacs Lisp library functions ***/
4833 Lisp_Object obj;
4834 {
4839 /* Get coding-spec vector for OBJ. */
4843 }
4849 Lisp_Object prompt;
4850 {
4851 Lisp_Object val;
4852 do
4853 {
4856 }
4859 }
4866 {
4867 Lisp_Object val;
4874 }
4883 Lisp_Object coding_system;
4884 {
4890 }
4891 \f
4892 Lisp_Object
4896 {
4907 {
4910 {
4911 Lisp_Object val2;
4915 }
4917 }
4919 /* At first, gather possible coding systems in VAL. */
4922 {
4923 int idx
4926 {
4930 }
4931 }
4935 /* Then, replace the elements with subsidiary coding systems. */
4937 {
4940 {
4941 Lisp_Object eol;
4945 }
4946 }
4948 }
4963 {
4981 }
4996 {
5002 }
5004 Lisp_Object
5008 {
5031 }
5044 {
5046 }
5059 {
5061 }
5063 Lisp_Object
5067 {
5082 }
5094 {
5096 }
5108 {
5110 }
5112 /* Encode or decode STRING according to CODING_SYSTEM.
5113 Do not set Vlast_coding_system_used. */
5115 Lisp_Object
5119 {
5133 }
5134 \f
5139 Lisp_Object code;
5140 {
5142 Lisp_Object val;
5147 {
5153 else
5155 }
5156 else
5157 {
5163 }
5165 }
5171 Lisp_Object ch;
5172 {
5174 Lisp_Object val;
5179 {
5181 }
5184 {
5187 }
5190 {
5192 }
5193 else
5196 }
5202 Lisp_Object code;
5203 {
5206 Lisp_Object val;
5211 {
5215 }
5216 else
5217 {
5223 }
5225 }
5231 Lisp_Object ch;
5232 {
5234 Lisp_Object val;
5239 {
5241 }
5246 {
5249 }
5250 else
5253 }
5254 \f
5256 Fset_terminal_coding_system_internal,
5259 Lisp_Object coding_system;
5260 {
5263 /* We had better not send unsafe characters to terminal. */
5267 }
5270 Fset_safe_terminal_coding_system_internal,
5273 Lisp_Object coding_system;
5274 {
5279 }
5284 ()
5285 {
5287 }
5290 Fset_keyboard_coding_system_internal,
5293 Lisp_Object coding_system;
5294 {
5298 }
5303 ()
5304 {
5306 }
5308 \f
5342 {
5362 ? Vfile_coding_system_alist
5364 ? Vnetwork_coding_system_alist
5370 {
5371 Lisp_Object elt;
5379 {
5381 /* Here, if VAL is both a valid coding system and a valid
5382 function symbol, we return VAL as a coding system. */
5390 {
5396 }
5398 }
5399 }
5401 }
5412 ()
5413 {
5417 {
5418 Lisp_Object val;
5422 {
5427 }
5429 {
5432 }
5433 }
5436 }
5442 ()
5443 {
5445 Lisp_Object val;
5450 {
5458 }
5459 /* If coding-category-list is valid and contains all coding
5460 categories, `i' should be CODING_CATEGORY_IDX_MAX now. If not,
5461 the following code saves Emacs from craching. */
5466 }
5470 \f
5471 /*** 9. Post-amble ***/
5473 void
5475 {
5477 }
5479 void
5481 {
5484 /* Emacs' internal format specific initialize routine. */
5500 /* ISO2022 specific initialize routine. */
5533 #if defined (MSDOS) || defined (WINDOWSNT)
5535 #else
5537 #endif
5540 }
5542 #ifdef emacs
5544 void
5546 {
5554 /* Target FILENAME is the first argument. */
5556 /* Target FILENAME is the third argument. */
5561 /* Target PROGRAM is the first argument. */
5566 /* Target PROGRAM is the third argument. */
5571 /* Target PROGRAM is the third argument. */
5576 /* Target SERVICE is the fourth argument. */
5616 Vcoding_category_table
5619 {
5622 {
5627 }
5628 }
5698 {
5703 Vcoding_category_list
5705 Vcoding_category_list);
5706 }
5863 }
5868 {
5875 {
5877 Vlocale_coding_system,
5880 }
5883 }