1 /* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Library General Public License as
8 published by the Free Software Foundation; either version 2 of the
9 License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Library General Public License for more details.
16 You should have received a copy of the GNU Library General Public
17 License along with the GNU C Library; see the file COPYING.LIB. If not,
18 write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 Boston, MA 02111-1307, USA. */
33 #include "iconv_prog.h"
36 /* Prototypes for a few program-wide used functions. */
37 extern void *xmalloc (size_t __n
);
38 extern void *xcalloc (size_t __n
, size_t __s
);
46 struct convtable
*sub
;
52 static inline struct convtable
*
55 return (struct convtable
*) xcalloc (1, sizeof (struct convtable
));
60 is_term (struct convtable
*tbl
, unsigned int idx
)
62 return tbl
->term
[idx
/ 8] & (1 << (idx
% 8));
67 clear_term (struct convtable
*tbl
, unsigned int idx
)
69 tbl
->term
[idx
/ 8] &= ~(1 << (idx
% 8));
74 set_term (struct convtable
*tbl
, unsigned int idx
)
76 tbl
->term
[idx
/ 8] |= 1 << (idx
% 8);
80 /* Generate the conversion table. */
81 static struct convtable
*use_from_charmap (struct charmap_t
*from_charmap
,
83 static struct convtable
*use_to_charmap (const char *from_code
,
84 struct charmap_t
*to_charmap
);
85 static struct convtable
*use_both_charmaps (struct charmap_t
*from_charmap
,
86 struct charmap_t
*to_charmap
);
88 /* Prototypes for the functions doing the actual work. */
89 static int process_block (struct convtable
*tbl
, char *addr
, size_t len
,
91 static int process_fd (struct convtable
*tbl
, int fd
, FILE *output
);
92 static int process_file (struct convtable
*tbl
, FILE *input
, FILE *output
);
96 charmap_conversion (const char *from_code
, struct charmap_t
*from_charmap
,
97 const char *to_code
, struct charmap_t
*to_charmap
,
98 int argc
, int remaining
, char *argv
[], FILE *output
)
100 struct convtable
*cvtbl
;
101 int status
= EXIT_SUCCESS
;
103 /* We have three different cases to handle:
105 - both, from_charmap and to_charmap, are available. This means we
106 can assume that the symbolic names match and use them to create
109 - only from_charmap is available. In this case we can only hope that
110 the symbolic names used are of the <Uxxxx> form in which case we
111 can use a UCS4->"to_code" iconv() conversion for the second step.
113 - only to_charmap is available. This is similar, only that we would
114 use iconv() for the "to_code"->UCS4 conversion.
116 We first create a table which maps input bytes into output bytes.
117 Once this is done we can handle all three of the cases above
119 if (from_charmap
!= NULL
)
121 if (to_charmap
== NULL
)
122 cvtbl
= use_from_charmap (from_charmap
, to_code
);
124 cvtbl
= use_both_charmaps (from_charmap
, to_charmap
);
128 assert (to_charmap
!= NULL
);
129 cvtbl
= use_to_charmap (from_code
, to_charmap
);
132 /* If we couldn't generate a table stop now. */
136 /* We can now start the conversion. */
137 if (remaining
== argc
)
139 if (process_file (cvtbl
, stdin
, output
) != 0)
140 status
= EXIT_FAILURE
;
150 printf ("%s:\n", argv
[remaining
]);
151 if (strcmp (argv
[remaining
], "-") == 0)
155 fd
= open (argv
[remaining
], O_RDONLY
);
159 error (0, errno
, _("cannot open input file `%s'"),
161 status
= EXIT_FAILURE
;
166 #ifdef _POSIX_MAPPED_FILES
167 /* We have possibilities for reading the input file. First try
168 to mmap() it since this will provide the fastest solution. */
169 if (fstat (fd
, &st
) == 0
170 && ((addr
= mmap (NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
,
171 fd
, 0)) != MAP_FAILED
))
173 /* Yes, we can use mmap(). The descriptor is not needed
176 error (EXIT_FAILURE
, errno
,
177 _("error while closing input `%s'"), argv
[remaining
]);
179 if (process_block (cvtbl
, addr
, st
.st_size
, output
) < 0)
181 /* Something went wrong. */
182 status
= EXIT_FAILURE
;
184 /* We don't need the input data anymore. */
185 munmap ((void *) addr
, st
.st_size
);
187 /* We cannot go on with producing output since it might
188 lead to problem because the last output might leave
189 the output stream in an undefined state. */
193 /* We don't need the input data anymore. */
194 munmap ((void *) addr
, st
.st_size
);
197 #endif /* _POSIX_MAPPED_FILES */
199 /* Read the file in pieces. */
200 if (process_fd (cvtbl
, fd
, output
) != 0)
202 /* Something went wrong. */
203 status
= EXIT_FAILURE
;
205 /* We don't need the input file anymore. */
208 /* We cannot go on with producing output since it might
209 lead to problem because the last output might leave
210 the output stream in an undefined state. */
214 /* Now close the file. */
218 while (++remaining
< argc
);
226 add_bytes (struct convtable
*tbl
, struct charseq
*in
, struct charseq
*out
)
231 assert (in
->nbytes
> 0);
233 byte
= ((unsigned char *) in
->bytes
)[n
];
234 while (n
+ 1 < in
->nbytes
)
236 if (is_term (tbl
, byte
) || tbl
->val
[byte
].sub
== NULL
)
238 /* Note that we simply ignore a definition for a byte sequence
239 which is also the prefix for a longer one. */
240 clear_term (tbl
, byte
);
242 (struct convtable
*) xcalloc (1, sizeof (struct convtable
));
245 tbl
= tbl
->val
[byte
].sub
;
247 byte
= ((unsigned char *) in
->bytes
)[++n
];
250 /* Only add the new sequence if there is none yet and the byte sequence
251 is not part of an even longer one. */
252 if (! is_term (tbl
, byte
) && tbl
->val
[byte
].sub
== NULL
)
254 set_term (tbl
, byte
);
255 tbl
->val
[byte
].out
= out
;
260 static struct convtable
*
261 use_from_charmap (struct charmap_t
*from_charmap
, const char *to_code
)
263 /* We iterate over all entries in the from_charmap and for those which
264 have a known UCS4 representation we use an iconv() call to determine
265 the mapping to the to_code charset. */
266 struct convtable
*rettbl
;
273 cd
= iconv_open (to_code
, "WCHAR_T");
274 if (cd
== (iconv_t
) -1)
275 /* We cannot do anything. */
278 rettbl
= allocate_table ();
280 while (iterate_table (&from_charmap
->char_table
, &ptr
, &key
, &keylen
, &data
)
283 struct charseq
*in
= (struct charseq
*) data
;
285 if (in
->ucs4
!= UNINITIALIZED_CHAR_VALUE
)
287 /* There is a chance. Try the iconv module. */
288 wchar_t inbuf
[1] = { in
->ucs4
};
289 unsigned char outbuf
[64];
290 char *inptr
= (char *) inbuf
;
291 size_t inlen
= sizeof (inbuf
);
292 char *outptr
= (char *) outbuf
;
293 size_t outlen
= sizeof (outbuf
);
295 (void) iconv (cd
, &inptr
, &inlen
, &outptr
, &outlen
);
297 if (outptr
!= (char *) outbuf
)
299 /* We got some output. Good, use it. */
300 struct charseq
*newp
;
302 outlen
= sizeof (outbuf
) - outlen
;
303 assert ((char *) outbuf
+ outlen
== outptr
);
305 newp
= (struct charseq
*) xmalloc (sizeof (struct charseq
)
307 newp
->name
= in
->name
;
308 newp
->ucs4
= in
->ucs4
;
309 newp
->nbytes
= outlen
;
310 memcpy (newp
->bytes
, outbuf
, outlen
);
312 add_bytes (rettbl
, in
, newp
);
315 /* Clear any possible state left behind. */
316 (void) iconv (cd
, NULL
, NULL
, NULL
, NULL
);
326 static struct convtable
*
327 use_to_charmap (const char *from_code
, struct charmap_t
*to_charmap
)
329 /* We iterate over all entries in the to_charmap and for those which
330 have a known UCS4 representation we use an iconv() call to determine
331 the mapping to the from_code charset. */
332 struct convtable
*rettbl
;
339 /* Note that the conversion we use here is the reverse direction. Without
340 exhaustive search we cannot figure out which input yields the UCS4
341 character we are looking for. Therefore we determine it the other
343 cd
= iconv_open (from_code
, "WCHAR_T");
344 if (cd
== (iconv_t
) -1)
345 /* We cannot do anything. */
348 rettbl
= allocate_table ();
350 while (iterate_table (&to_charmap
->char_table
, &ptr
, &key
, &keylen
, &data
)
353 struct charseq
*out
= (struct charseq
*) data
;
355 if (out
->ucs4
!= UNINITIALIZED_CHAR_VALUE
)
357 /* There is a chance. Try the iconv module. */
358 wchar_t inbuf
[1] = { out
->ucs4
};
359 unsigned char outbuf
[64];
360 char *inptr
= (char *) inbuf
;
361 size_t inlen
= sizeof (inbuf
);
362 char *outptr
= (char *) outbuf
;
363 size_t outlen
= sizeof (outbuf
);
365 (void) iconv (cd
, &inptr
, &inlen
, &outptr
, &outlen
);
367 if (outptr
!= (char *) outbuf
)
369 /* We got some output. Good, use it. */
370 struct charseq
*newp
;
372 outlen
= sizeof (outbuf
) - outlen
;
373 assert ((char *) outbuf
+ outlen
== outptr
);
375 newp
= (struct charseq
*) xmalloc (sizeof (struct charseq
)
377 newp
->name
= out
->name
;
378 newp
->ucs4
= out
->ucs4
;
379 newp
->nbytes
= outlen
;
380 memcpy (newp
->bytes
, outbuf
, outlen
);
382 add_bytes (rettbl
, newp
, out
);
385 /* Clear any possible state left behind. */
386 (void) iconv (cd
, NULL
, NULL
, NULL
, NULL
);
396 static struct convtable
*
397 use_both_charmaps (struct charmap_t
*from_charmap
,
398 struct charmap_t
*to_charmap
)
400 /* In this case we iterate over all the entries in the from_charmap,
401 determine the internal name, and find an appropriate entry in the
402 to_charmap (if it exists). */
403 struct convtable
*rettbl
= allocate_table ();
409 while (iterate_table (&from_charmap
->char_table
, &ptr
, &key
, &keylen
, &data
)
412 struct charseq
*in
= (struct charseq
*) data
;
413 struct charseq
*out
= charmap_find_value (to_charmap
, key
, keylen
);
416 add_bytes (rettbl
, in
, out
);
424 process_block (struct convtable
*tbl
, char *addr
, size_t len
, FILE *output
)
430 struct convtable
*cur
= tbl
;
431 unsigned char *curp
= (unsigned char *) addr
;
432 unsigned int byte
= *curp
;
436 while (! is_term (cur
, byte
))
437 if (cur
->val
[byte
].sub
== NULL
)
439 /* This is a invalid sequence. Skip the first byte if we are
440 ignoring errors. Otherwise punt. */
443 error (0, 0, _("illegal input sequence at position %Zd"), n
);
447 n
-= curp
- (unsigned char *) addr
;
449 byte
= *(curp
= (unsigned char *) ++addr
);
458 cur
= cur
->val
[byte
].sub
;
463 incomplete character or shift sequence at end of buffer"));
470 /* We found a final byte. Write the output bytes. */
471 out
= cur
->val
[byte
].out
;
472 for (cnt
= 0; cnt
< out
->nbytes
; ++cnt
)
473 fputc_unlocked (out
->bytes
[cnt
], output
);
475 addr
= (char *) curp
+ 1;
484 process_fd (struct convtable
*tbl
, int fd
, FILE *output
)
486 /* we have a problem with reading from a desriptor since we must not
487 provide the iconv() function an incomplete character or shift
488 sequence at the end of the buffer. Since we have to deal with
489 arbitrary encodings we must read the whole text in a buffer and
490 process it in one step. */
491 static char *inbuf
= NULL
;
492 static size_t maxlen
= 0;
496 while (actlen
< maxlen
)
498 ssize_t n
= read (fd
, inptr
, maxlen
- actlen
);
501 /* No more text to read. */
506 /* Error while reading. */
507 error (0, errno
, _("error while reading the input"));
515 if (actlen
== maxlen
)
520 /* Increase the buffer. */
522 inbuf
= realloc (inbuf
, maxlen
);
524 error (0, errno
, _("unable to allocate buffer for input"));
525 inptr
= inbuf
+ actlen
;
529 n
= read (fd
, inptr
, maxlen
- actlen
);
532 /* No more text to read. */
537 /* Error while reading. */
538 error (0, errno
, _("error while reading the input"));
545 while (actlen
< maxlen
);
548 /* Break again so we leave both loops. */
552 /* Now we have all the input in the buffer. Process it in one run. */
553 return process_block (tbl
, inbuf
, actlen
, output
);
558 process_file (struct convtable
*tbl
, FILE *input
, FILE *output
)
560 /* This should be safe since we use this function only for `stdin' and
561 we haven't read anything so far. */
562 return process_fd (tbl
, fileno (input
), output
);