Update.
[glibc.git] / iconv / iconv_charmap.c
blob4b72b1bad22804176efc00c4e43a4e8aad479794
1 /* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Library General Public License as
8 published by the Free Software Foundation; either version 2 of the
9 License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Library General Public License for more details.
16 You should have received a copy of the GNU Library General Public
17 License along with the GNU C Library; see the file COPYING.LIB. If not,
18 write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 Boston, MA 02111-1307, USA. */
21 #include <assert.h>
22 #include <errno.h>
23 #include <error.h>
24 #include <fcntl.h>
25 #include <iconv.h>
26 #include <libintl.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <unistd.h>
30 #include <sys/mman.h>
31 #include <sys/stat.h>
33 #include "iconv_prog.h"
36 /* Prototypes for a few program-wide used functions. */
37 extern void *xmalloc (size_t __n);
38 extern void *xcalloc (size_t __n, size_t __s);
41 struct convtable
43 int term[256 / 8];
44 union
46 struct convtable *sub;
47 struct charseq *out;
48 } val[256];
52 static inline struct convtable *
53 allocate_table (void)
55 return (struct convtable *) xcalloc (1, sizeof (struct convtable));
59 static inline int
60 is_term (struct convtable *tbl, unsigned int idx)
62 return tbl->term[idx / 8] & (1 << (idx % 8));
66 static inline void
67 clear_term (struct convtable *tbl, unsigned int idx)
69 tbl->term[idx / 8] &= ~(1 << (idx % 8));
73 static inline void
74 set_term (struct convtable *tbl, unsigned int idx)
76 tbl->term[idx / 8] |= 1 << (idx % 8);
80 /* Generate the conversion table. */
81 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
82 const char *to_code);
83 static struct convtable *use_to_charmap (const char *from_code,
84 struct charmap_t *to_charmap);
85 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
86 struct charmap_t *to_charmap);
88 /* Prototypes for the functions doing the actual work. */
89 static int process_block (struct convtable *tbl, char *addr, size_t len,
90 FILE *output);
91 static int process_fd (struct convtable *tbl, int fd, FILE *output);
92 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
95 int
96 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
97 const char *to_code, struct charmap_t *to_charmap,
98 int argc, int remaining, char *argv[], FILE *output)
100 struct convtable *cvtbl;
101 int status = EXIT_SUCCESS;
103 /* We have three different cases to handle:
105 - both, from_charmap and to_charmap, are available. This means we
106 can assume that the symbolic names match and use them to create
107 the mapping.
109 - only from_charmap is available. In this case we can only hope that
110 the symbolic names used are of the <Uxxxx> form in which case we
111 can use a UCS4->"to_code" iconv() conversion for the second step.
113 - only to_charmap is available. This is similar, only that we would
114 use iconv() for the "to_code"->UCS4 conversion.
116 We first create a table which maps input bytes into output bytes.
117 Once this is done we can handle all three of the cases above
118 equally. */
119 if (from_charmap != NULL)
121 if (to_charmap == NULL)
122 cvtbl = use_from_charmap (from_charmap, to_code);
123 else
124 cvtbl = use_both_charmaps (from_charmap, to_charmap);
126 else
128 assert (to_charmap != NULL);
129 cvtbl = use_to_charmap (from_code, to_charmap);
132 /* If we couldn't generate a table stop now. */
133 if (cvtbl == NULL)
134 return EXIT_FAILURE;
136 /* We can now start the conversion. */
137 if (remaining == argc)
139 if (process_file (cvtbl, stdin, output) != 0)
140 status = EXIT_FAILURE;
142 else
145 struct stat st;
146 char *addr;
147 int fd;
149 if (verbose)
150 printf ("%s:\n", argv[remaining]);
151 if (strcmp (argv[remaining], "-") == 0)
152 fd = 0;
153 else
155 fd = open (argv[remaining], O_RDONLY);
157 if (fd == -1)
159 error (0, errno, _("cannot open input file `%s'"),
160 argv[remaining]);
161 status = EXIT_FAILURE;
162 continue;
166 #ifdef _POSIX_MAPPED_FILES
167 /* We have possibilities for reading the input file. First try
168 to mmap() it since this will provide the fastest solution. */
169 if (fstat (fd, &st) == 0
170 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
171 fd, 0)) != MAP_FAILED))
173 /* Yes, we can use mmap(). The descriptor is not needed
174 anymore. */
175 if (close (fd) != 0)
176 error (EXIT_FAILURE, errno,
177 _("error while closing input `%s'"), argv[remaining]);
179 if (process_block (cvtbl, addr, st.st_size, output) < 0)
181 /* Something went wrong. */
182 status = EXIT_FAILURE;
184 /* We don't need the input data anymore. */
185 munmap ((void *) addr, st.st_size);
187 /* We cannot go on with producing output since it might
188 lead to problem because the last output might leave
189 the output stream in an undefined state. */
190 break;
193 /* We don't need the input data anymore. */
194 munmap ((void *) addr, st.st_size);
196 else
197 #endif /* _POSIX_MAPPED_FILES */
199 /* Read the file in pieces. */
200 if (process_fd (cvtbl, fd, output) != 0)
202 /* Something went wrong. */
203 status = EXIT_FAILURE;
205 /* We don't need the input file anymore. */
206 close (fd);
208 /* We cannot go on with producing output since it might
209 lead to problem because the last output might leave
210 the output stream in an undefined state. */
211 break;
214 /* Now close the file. */
215 close (fd);
218 while (++remaining < argc);
220 /* All done. */
221 return status;
225 static void
226 add_bytes (struct convtable *tbl, struct charseq *in, struct charseq *out)
228 int n = 0;
229 unsigned int byte;
231 assert (in->nbytes > 0);
233 byte = ((unsigned char *) in->bytes)[n];
234 while (n + 1 < in->nbytes)
236 if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
238 /* Note that we simply ignore a definition for a byte sequence
239 which is also the prefix for a longer one. */
240 clear_term (tbl, byte);
241 tbl->val[byte].sub =
242 (struct convtable *) xcalloc (1, sizeof (struct convtable));
245 tbl = tbl->val[byte].sub;
247 byte = ((unsigned char *) in->bytes)[++n];
250 /* Only add the new sequence if there is none yet and the byte sequence
251 is not part of an even longer one. */
252 if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
254 set_term (tbl, byte);
255 tbl->val[byte].out = out;
260 static struct convtable *
261 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
263 /* We iterate over all entries in the from_charmap and for those which
264 have a known UCS4 representation we use an iconv() call to determine
265 the mapping to the to_code charset. */
266 struct convtable *rettbl;
267 iconv_t cd;
268 void *ptr = NULL;
269 const void *key;
270 size_t keylen;
271 void *data;
273 cd = iconv_open (to_code, "WCHAR_T");
274 if (cd == (iconv_t) -1)
275 /* We cannot do anything. */
276 return NULL;
278 rettbl = allocate_table ();
280 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
281 >= 0)
283 struct charseq *in = (struct charseq *) data;
285 if (in->ucs4 != UNINITIALIZED_CHAR_VALUE)
287 /* There is a chance. Try the iconv module. */
288 wchar_t inbuf[1] = { in->ucs4 };
289 unsigned char outbuf[64];
290 char *inptr = (char *) inbuf;
291 size_t inlen = sizeof (inbuf);
292 char *outptr = (char *) outbuf;
293 size_t outlen = sizeof (outbuf);
295 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
297 if (outptr != (char *) outbuf)
299 /* We got some output. Good, use it. */
300 struct charseq *newp;
302 outlen = sizeof (outbuf) - outlen;
303 assert ((char *) outbuf + outlen == outptr);
305 newp = (struct charseq *) xmalloc (sizeof (struct charseq)
306 + outlen);
307 newp->name = in->name;
308 newp->ucs4 = in->ucs4;
309 newp->nbytes = outlen;
310 memcpy (newp->bytes, outbuf, outlen);
312 add_bytes (rettbl, in, newp);
315 /* Clear any possible state left behind. */
316 (void) iconv (cd, NULL, NULL, NULL, NULL);
320 iconv_close (cd);
322 return rettbl;
326 static struct convtable *
327 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
329 /* We iterate over all entries in the to_charmap and for those which
330 have a known UCS4 representation we use an iconv() call to determine
331 the mapping to the from_code charset. */
332 struct convtable *rettbl;
333 iconv_t cd;
334 void *ptr = NULL;
335 const void *key;
336 size_t keylen;
337 void *data;
339 /* Note that the conversion we use here is the reverse direction. Without
340 exhaustive search we cannot figure out which input yields the UCS4
341 character we are looking for. Therefore we determine it the other
342 way round. */
343 cd = iconv_open (from_code, "WCHAR_T");
344 if (cd == (iconv_t) -1)
345 /* We cannot do anything. */
346 return NULL;
348 rettbl = allocate_table ();
350 while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
351 >= 0)
353 struct charseq *out = (struct charseq *) data;
355 if (out->ucs4 != UNINITIALIZED_CHAR_VALUE)
357 /* There is a chance. Try the iconv module. */
358 wchar_t inbuf[1] = { out->ucs4 };
359 unsigned char outbuf[64];
360 char *inptr = (char *) inbuf;
361 size_t inlen = sizeof (inbuf);
362 char *outptr = (char *) outbuf;
363 size_t outlen = sizeof (outbuf);
365 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
367 if (outptr != (char *) outbuf)
369 /* We got some output. Good, use it. */
370 struct charseq *newp;
372 outlen = sizeof (outbuf) - outlen;
373 assert ((char *) outbuf + outlen == outptr);
375 newp = (struct charseq *) xmalloc (sizeof (struct charseq)
376 + outlen);
377 newp->name = out->name;
378 newp->ucs4 = out->ucs4;
379 newp->nbytes = outlen;
380 memcpy (newp->bytes, outbuf, outlen);
382 add_bytes (rettbl, newp, out);
385 /* Clear any possible state left behind. */
386 (void) iconv (cd, NULL, NULL, NULL, NULL);
390 iconv_close (cd);
392 return rettbl;
396 static struct convtable *
397 use_both_charmaps (struct charmap_t *from_charmap,
398 struct charmap_t *to_charmap)
400 /* In this case we iterate over all the entries in the from_charmap,
401 determine the internal name, and find an appropriate entry in the
402 to_charmap (if it exists). */
403 struct convtable *rettbl = allocate_table ();
404 void *ptr = NULL;
405 const void *key;
406 size_t keylen;
407 void *data;
409 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
410 >= 0)
412 struct charseq *in = (struct charseq *) data;
413 struct charseq *out = charmap_find_value (to_charmap, key, keylen);
415 if (out != NULL)
416 add_bytes (rettbl, in, out);
419 return rettbl;
423 static int
424 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
426 size_t n = 0;
428 while (n < len)
430 struct convtable *cur = tbl;
431 unsigned char *curp = (unsigned char *) addr;
432 unsigned int byte = *curp;
433 int cnt;
434 struct charseq *out;
436 while (! is_term (cur, byte))
437 if (cur->val[byte].sub == NULL)
439 /* This is a invalid sequence. Skip the first byte if we are
440 ignoring errors. Otherwise punt. */
441 if (! omit_invalid)
443 error (0, 0, _("illegal input sequence at position %Zd"), n);
444 return -1;
447 n -= curp - (unsigned char *) addr;
449 byte = *(curp = (unsigned char *) ++addr);
450 if (++n >= len)
451 /* All converted. */
452 return 0;
454 cur = tbl;
456 else
458 cur = cur->val[byte].sub;
460 if (++n >= len)
462 error (0, 0, _("\
463 incomplete character or shift sequence at end of buffer"));
464 return -1;
467 byte = *++curp;
470 /* We found a final byte. Write the output bytes. */
471 out = cur->val[byte].out;
472 for (cnt = 0; cnt < out->nbytes; ++cnt)
473 fputc_unlocked (out->bytes[cnt], output);
475 addr = (char *) curp + 1;
476 ++n;
479 return 0;
483 static int
484 process_fd (struct convtable *tbl, int fd, FILE *output)
486 /* we have a problem with reading from a desriptor since we must not
487 provide the iconv() function an incomplete character or shift
488 sequence at the end of the buffer. Since we have to deal with
489 arbitrary encodings we must read the whole text in a buffer and
490 process it in one step. */
491 static char *inbuf = NULL;
492 static size_t maxlen = 0;
493 char *inptr = NULL;
494 size_t actlen = 0;
496 while (actlen < maxlen)
498 ssize_t n = read (fd, inptr, maxlen - actlen);
500 if (n == 0)
501 /* No more text to read. */
502 break;
504 if (n == -1)
506 /* Error while reading. */
507 error (0, errno, _("error while reading the input"));
508 return -1;
511 inptr += n;
512 actlen += n;
515 if (actlen == maxlen)
516 while (1)
518 ssize_t n;
520 /* Increase the buffer. */
521 maxlen += 32768;
522 inbuf = realloc (inbuf, maxlen);
523 if (inbuf == NULL)
524 error (0, errno, _("unable to allocate buffer for input"));
525 inptr = inbuf + actlen;
529 n = read (fd, inptr, maxlen - actlen);
531 if (n == 0)
532 /* No more text to read. */
533 break;
535 if (n == -1)
537 /* Error while reading. */
538 error (0, errno, _("error while reading the input"));
539 return -1;
542 inptr += n;
543 actlen += n;
545 while (actlen < maxlen);
547 if (n == 0)
548 /* Break again so we leave both loops. */
549 break;
552 /* Now we have all the input in the buffer. Process it in one run. */
553 return process_block (tbl, inbuf, actlen, output);
557 static int
558 process_file (struct convtable *tbl, FILE *input, FILE *output)
560 /* This should be safe since we use this function only for `stdin' and
561 we haven't read anything so far. */
562 return process_fd (tbl, fileno (input), output);