Replace FSF snail mail address with URLs.
[glibc.git] / iconv / iconv_prog.c
blob36168a1a919f4ca66c800b4580f9969e93f70db2
1 /* Convert text in given files from the specified from-set to the to-set.
2 Copyright (C) 1998-2011, 2012 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>. */
19 #include <argp.h>
20 #include <assert.h>
21 #include <ctype.h>
22 #include <errno.h>
23 #include <error.h>
24 #include <fcntl.h>
25 #include <iconv.h>
26 #include <langinfo.h>
27 #include <locale.h>
28 #include <search.h>
29 #include <stdbool.h>
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <unistd.h>
34 #include <libintl.h>
35 #ifdef _POSIX_MAPPED_FILES
36 # include <sys/mman.h>
37 #endif
38 #include <charmap.h>
39 #include <gconv_int.h>
40 #include "iconv_prog.h"
41 #include "iconvconfig.h"
43 /* Get libc version number. */
44 #include "../version.h"
46 #define PACKAGE _libc_intl_domainname
49 /* Name and version of program. */
50 static void print_version (FILE *stream, struct argp_state *state);
51 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
53 #define OPT_VERBOSE 1000
54 #define OPT_LIST 'l'
56 /* Definitions of arguments for argp functions. */
57 static const struct argp_option options[] =
59 { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
60 { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
61 { "to-code", 't', "NAME", 0, N_("encoding for output") },
62 { NULL, 0, NULL, 0, N_("Information:") },
63 { "list", 'l', NULL, 0, N_("list all known coded character sets") },
64 { NULL, 0, NULL, 0, N_("Output control:") },
65 { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
66 { "output", 'o', "FILE", 0, N_("output file") },
67 { "silent", 's', NULL, 0, N_("suppress warnings") },
68 { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
69 { NULL, 0, NULL, 0, NULL }
72 /* Short description of program. */
73 static const char doc[] = N_("\
74 Convert encoding of given files from one encoding to another.");
76 /* Strings for arguments in help texts. */
77 static const char args_doc[] = N_("[FILE...]");
79 /* Prototype for option handler. */
80 static error_t parse_opt (int key, char *arg, struct argp_state *state);
82 /* Function to print some extra text in the help message. */
83 static char *more_help (int key, const char *text, void *input);
85 /* Data structure to communicate with argp functions. */
86 static struct argp argp =
88 options, parse_opt, args_doc, doc, NULL, more_help
91 /* Code sets to convert from and to respectively. An empty string as the
92 default causes the 'iconv_open' function to look up the charset of the
93 currently selected locale and use it. */
94 static const char *from_code = "";
95 static const char *to_code = "";
97 /* File to write output to. If NULL write to stdout. */
98 static const char *output_file;
100 /* Nonzero if verbose ouput is wanted. */
101 int verbose;
103 /* Nonzero if list of all coded character sets is wanted. */
104 static int list;
106 /* If nonzero omit invalid character from output. */
107 int omit_invalid;
109 /* Prototypes for the functions doing the actual work. */
110 static int process_block (iconv_t cd, char *addr, size_t len, FILE **output,
111 const char *output_file);
112 static int process_fd (iconv_t cd, int fd, FILE **output,
113 const char *output_file);
114 static int process_file (iconv_t cd, FILE *input, FILE **output,
115 const char *output_file);
116 static void print_known_names (void) internal_function;
120 main (int argc, char *argv[])
122 int status = EXIT_SUCCESS;
123 int remaining;
124 iconv_t cd;
125 const char *orig_to_code;
126 struct charmap_t *from_charmap = NULL;
127 struct charmap_t *to_charmap = NULL;
129 /* Set locale via LC_ALL. */
130 setlocale (LC_ALL, "");
132 /* Set the text message domain. */
133 textdomain (_libc_intl_domainname);
135 /* Parse and process arguments. */
136 argp_parse (&argp, argc, argv, 0, &remaining, NULL);
138 /* List all coded character sets if wanted. */
139 if (list)
141 print_known_names ();
142 exit (EXIT_SUCCESS);
145 /* If we have to ignore errors make sure we use the appropriate name for
146 the to-character-set. */
147 orig_to_code = to_code;
148 if (omit_invalid)
150 const char *errhand = strchrnul (to_code, '/');
151 int nslash = 2;
152 char *newp;
153 char *cp;
155 if (*errhand == '/')
157 --nslash;
158 errhand = strchrnul (errhand, '/');
160 if (*errhand == '/')
162 --nslash;
163 errhand = strchr (errhand, '\0');
167 newp = (char *) alloca (errhand - to_code + nslash + 7 + 1);
168 cp = mempcpy (newp, to_code, errhand - to_code);
169 while (nslash-- > 0)
170 *cp++ = '/';
171 if (cp[-1] != '/')
172 *cp++ = ',';
173 memcpy (cp, "IGNORE", sizeof ("IGNORE"));
175 to_code = newp;
178 /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
179 can be file names of charmaps. In this case iconv will have to read
180 those charmaps and use them to do the conversion. But there are
181 holes in the specification. There is nothing said that if -f is a
182 charmap filename that -t must be, too. And vice versa. There is
183 also no word about the symbolic names used. What if they don't
184 match? */
185 if (strchr (from_code, '/') != NULL)
186 /* The from-name might be a charmap file name. Try reading the
187 file. */
188 from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0, 0);
190 if (strchr (orig_to_code, '/') != NULL)
191 /* The to-name might be a charmap file name. Try reading the
192 file. */
193 to_charmap = charmap_read (orig_to_code, /*0, 1,*/1, 0, 0, 0);
196 /* At this point we have to handle two cases. The first one is
197 where a charmap is used for the from- or to-charset, or both. We
198 handle this special since it is very different from the sane way of
199 doing things. The other case allows converting using the iconv()
200 function. */
201 if (from_charmap != NULL || to_charmap != NULL)
202 /* Construct the conversion table and do the conversion. */
203 status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
204 argc, remaining, argv, output_file);
205 else
207 /* Let's see whether we have these coded character sets. */
208 cd = iconv_open (to_code, from_code);
209 if (cd == (iconv_t) -1)
211 if (errno == EINVAL)
213 /* Try to be nice with the user and tell her which of the
214 two encoding names is wrong. This is possible because
215 all supported encodings can be converted from/to Unicode,
216 in other words, because the graph of encodings is
217 connected. */
218 bool from_wrong =
219 (iconv_open ("UTF-8", from_code) == (iconv_t) -1
220 && errno == EINVAL);
221 bool to_wrong =
222 (iconv_open (to_code, "UTF-8") == (iconv_t) -1
223 && errno == EINVAL);
224 const char *from_pretty =
225 (from_code[0] ? from_code : nl_langinfo (CODESET));
226 const char *to_pretty =
227 (orig_to_code[0] ? orig_to_code : nl_langinfo (CODESET));
229 if (from_wrong)
231 if (to_wrong)
232 error (0, 0,
233 _("\
234 conversions from `%s' and to `%s' are not supported"),
235 from_pretty, to_pretty);
236 else
237 error (0, 0,
238 _("conversion from `%s' is not supported"),
239 from_pretty);
241 else
243 if (to_wrong)
244 error (0, 0,
245 _("conversion to `%s' is not supported"),
246 to_pretty);
247 else
248 error (0, 0,
249 _("conversion from `%s' to `%s' is not supported"),
250 from_pretty, to_pretty);
253 argp_help (&argp, stderr, ARGP_HELP_SEE,
254 program_invocation_short_name);
255 exit (1);
257 else
258 error (EXIT_FAILURE, errno,
259 _("failed to start conversion processing"));
262 /* The output file. Will be opened when we are ready to produce
263 output. */
264 FILE *output = NULL;
266 /* Now process the remaining files. Write them to stdout or the file
267 specified with the `-o' parameter. If we have no file given as
268 the parameter process all from stdin. */
269 if (remaining == argc)
271 if (process_file (cd, stdin, &output, output_file) != 0)
272 status = EXIT_FAILURE;
274 else
277 #ifdef _POSIX_MAPPED_FILES
278 struct stat st;
279 char *addr;
280 #endif
281 int fd, ret;
283 if (verbose)
284 fprintf (stderr, "%s:\n", argv[remaining]);
285 if (strcmp (argv[remaining], "-") == 0)
286 fd = 0;
287 else
289 fd = open (argv[remaining], O_RDONLY);
291 if (fd == -1)
293 error (0, errno, _("cannot open input file `%s'"),
294 argv[remaining]);
295 status = EXIT_FAILURE;
296 continue;
300 #ifdef _POSIX_MAPPED_FILES
301 /* We have possibilities for reading the input file. First try
302 to mmap() it since this will provide the fastest solution. */
303 if (fstat (fd, &st) == 0
304 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
305 fd, 0)) != MAP_FAILED))
307 /* Yes, we can use mmap(). The descriptor is not needed
308 anymore. */
309 if (close (fd) != 0)
310 error (EXIT_FAILURE, errno,
311 _("error while closing input `%s'"),
312 argv[remaining]);
314 ret = process_block (cd, addr, st.st_size, &output,
315 output_file);
317 /* We don't need the input data anymore. */
318 munmap ((void *) addr, st.st_size);
320 if (ret != 0)
322 status = EXIT_FAILURE;
324 if (ret < 0)
325 /* We cannot go on with producing output since it might
326 lead to problem because the last output might leave
327 the output stream in an undefined state. */
328 break;
331 else
332 #endif /* _POSIX_MAPPED_FILES */
334 /* Read the file in pieces. */
335 ret = process_fd (cd, fd, &output, output_file);
337 /* Now close the file. */
338 close (fd);
340 if (ret != 0)
342 /* Something went wrong. */
343 status = EXIT_FAILURE;
345 if (ret < 0)
346 /* We cannot go on with producing output since it might
347 lead to problem because the last output might leave
348 the output stream in an undefined state. */
349 break;
353 while (++remaining < argc);
355 /* Close the output file now. */
356 if (output != NULL && fclose (output))
357 error (EXIT_FAILURE, errno, _("error while closing output file"));
360 return status;
364 /* Handle program arguments. */
365 static error_t
366 parse_opt (int key, char *arg, struct argp_state *state)
368 switch (key)
370 case 'f':
371 from_code = arg;
372 break;
373 case 't':
374 to_code = arg;
375 break;
376 case 'o':
377 output_file = arg;
378 break;
379 case 's':
380 /* Nothing, for now at least. We are not giving out any information
381 about missing character or so. */
382 break;
383 case 'c':
384 /* Omit invalid characters from output. */
385 omit_invalid = 1;
386 break;
387 case OPT_VERBOSE:
388 verbose = 1;
389 break;
390 case OPT_LIST:
391 list = 1;
392 break;
393 default:
394 return ARGP_ERR_UNKNOWN;
396 return 0;
400 static char *
401 more_help (int key, const char *text, void *input)
403 switch (key)
405 case ARGP_KEY_HELP_EXTRA:
406 /* We print some extra information. */
407 return strdup (gettext ("\
408 For bug reporting instructions, please see:\n\
409 <http://www.gnu.org/software/libc/bugs.html>.\n"));
410 default:
411 break;
413 return (char *) text;
417 /* Print the version information. */
418 static void
419 print_version (FILE *stream, struct argp_state *state)
421 fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
422 fprintf (stream, gettext ("\
423 Copyright (C) %s Free Software Foundation, Inc.\n\
424 This is free software; see the source for copying conditions. There is NO\n\
425 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
426 "), "2012");
427 fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
431 static int
432 write_output (const char *outbuf, const char *outptr, FILE **output,
433 const char *output_file)
435 /* We have something to write out. */
436 int errno_save = errno;
438 if (*output == NULL)
440 /* Determine output file. */
441 if (output_file != NULL && strcmp (output_file, "-") != 0)
443 *output = fopen (output_file, "w");
444 if (*output == NULL)
445 error (EXIT_FAILURE, errno, _("cannot open output file"));
447 else
448 *output = stdout;
451 if (fwrite (outbuf, 1, outptr - outbuf, *output) < (size_t) (outptr - outbuf)
452 || ferror (*output))
454 /* Error occurred while printing the result. */
455 error (0, 0, _("\
456 conversion stopped due to problem in writing the output"));
457 return -1;
460 errno = errno_save;
462 return 0;
466 static int
467 process_block (iconv_t cd, char *addr, size_t len, FILE **output,
468 const char *output_file)
470 #define OUTBUF_SIZE 32768
471 const char *start = addr;
472 char outbuf[OUTBUF_SIZE];
473 char *outptr;
474 size_t outlen;
475 size_t n;
476 int ret = 0;
478 while (len > 0)
480 outptr = outbuf;
481 outlen = OUTBUF_SIZE;
482 n = iconv (cd, &addr, &len, &outptr, &outlen);
484 if (n == (size_t) -1 && omit_invalid && errno == EILSEQ)
486 ret = 1;
487 if (len == 0)
488 n = 0;
489 else
490 errno = E2BIG;
493 if (outptr != outbuf)
495 ret = write_output (outbuf, outptr, output, output_file);
496 if (ret != 0)
497 break;
500 if (n != (size_t) -1)
502 /* All the input test is processed. For state-dependent
503 character sets we have to flush the state now. */
504 outptr = outbuf;
505 outlen = OUTBUF_SIZE;
506 n = iconv (cd, NULL, NULL, &outptr, &outlen);
508 if (outptr != outbuf)
510 ret = write_output (outbuf, outptr, output, output_file);
511 if (ret != 0)
512 break;
515 if (n != (size_t) -1)
516 break;
518 if (omit_invalid && errno == EILSEQ)
520 ret = 1;
521 break;
525 if (errno != E2BIG)
527 /* iconv() ran into a problem. */
528 switch (errno)
530 case EILSEQ:
531 if (! omit_invalid)
532 error (0, 0, _("illegal input sequence at position %ld"),
533 (long int) (addr - start));
534 break;
535 case EINVAL:
536 error (0, 0, _("\
537 incomplete character or shift sequence at end of buffer"));
538 break;
539 case EBADF:
540 error (0, 0, _("internal error (illegal descriptor)"));
541 break;
542 default:
543 error (0, 0, _("unknown iconv() error %d"), errno);
544 break;
547 return -1;
551 return ret;
555 static int
556 process_fd (iconv_t cd, int fd, FILE **output, const char *output_file)
558 /* we have a problem with reading from a desriptor since we must not
559 provide the iconv() function an incomplete character or shift
560 sequence at the end of the buffer. Since we have to deal with
561 arbitrary encodings we must read the whole text in a buffer and
562 process it in one step. */
563 static char *inbuf = NULL;
564 static size_t maxlen = 0;
565 char *inptr = NULL;
566 size_t actlen = 0;
568 while (actlen < maxlen)
570 ssize_t n = read (fd, inptr, maxlen - actlen);
572 if (n == 0)
573 /* No more text to read. */
574 break;
576 if (n == -1)
578 /* Error while reading. */
579 error (0, errno, _("error while reading the input"));
580 return -1;
583 inptr += n;
584 actlen += n;
587 if (actlen == maxlen)
588 while (1)
590 ssize_t n;
591 char *new_inbuf;
593 /* Increase the buffer. */
594 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
595 if (new_inbuf == NULL)
597 error (0, errno, _("unable to allocate buffer for input"));
598 return -1;
600 inbuf = new_inbuf;
601 maxlen += 32768;
602 inptr = inbuf + actlen;
606 n = read (fd, inptr, maxlen - actlen);
608 if (n == 0)
609 /* No more text to read. */
610 break;
612 if (n == -1)
614 /* Error while reading. */
615 error (0, errno, _("error while reading the input"));
616 return -1;
619 inptr += n;
620 actlen += n;
622 while (actlen < maxlen);
624 if (n == 0)
625 /* Break again so we leave both loops. */
626 break;
629 /* Now we have all the input in the buffer. Process it in one run. */
630 return process_block (cd, inbuf, actlen, output, output_file);
634 static int
635 process_file (iconv_t cd, FILE *input, FILE **output, const char *output_file)
637 /* This should be safe since we use this function only for `stdin' and
638 we haven't read anything so far. */
639 return process_fd (cd, fileno (input), output, output_file);
643 /* Print all known character sets/encodings. */
644 static void *printlist;
645 static size_t column;
646 static int not_first;
648 static void
649 insert_print_list (const void *nodep, VISIT value, int level)
651 if (value == leaf || value == postorder)
653 const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
654 tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
658 static void
659 do_print_human (const void *nodep, VISIT value, int level)
661 if (value == leaf || value == postorder)
663 const char *s = *(const char **) nodep;
664 size_t len = strlen (s);
665 size_t cnt;
667 while (len > 0 && s[len - 1] == '/')
668 --len;
670 for (cnt = 0; cnt < len; ++cnt)
671 if (isalnum (s[cnt]))
672 break;
673 if (cnt == len)
674 return;
676 if (not_first)
678 putchar (',');
679 ++column;
681 if (column > 2 && column + len > 77)
683 fputs ("\n ", stdout);
684 column = 2;
686 else
688 putchar (' ');
689 ++column;
692 else
693 not_first = 1;
695 fwrite (s, len, 1, stdout);
696 column += len;
700 static void
701 do_print (const void *nodep, VISIT value, int level)
703 if (value == leaf || value == postorder)
705 const char *s = *(const char **) nodep;
707 puts (s);
711 static void
712 internal_function
713 add_known_names (struct gconv_module *node)
715 if (node->left != NULL)
716 add_known_names (node->left);
717 if (node->right != NULL)
718 add_known_names (node->right);
721 if (strcmp (node->from_string, "INTERNAL") != 0)
722 tsearch (node->from_string, &printlist, (__compar_fn_t) strverscmp);
723 if (strcmp (node->to_string, "INTERNAL") != 0)
724 tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
726 node = node->same;
728 while (node != NULL);
732 static void
733 insert_cache (void)
735 const struct gconvcache_header *header;
736 const char *strtab;
737 const struct hash_entry *hashtab;
738 size_t cnt;
740 header = (const struct gconvcache_header *) __gconv_get_cache ();
741 strtab = (char *) header + header->string_offset;
742 hashtab = (struct hash_entry *) ((char *) header + header->hash_offset);
744 for (cnt = 0; cnt < header->hash_size; ++cnt)
745 if (hashtab[cnt].string_offset != 0)
747 const char *str = strtab + hashtab[cnt].string_offset;
749 if (strcmp (str, "INTERNAL") != 0)
750 tsearch (str, &printlist, (__compar_fn_t) strverscmp);
755 static void
756 internal_function
757 print_known_names (void)
759 iconv_t h;
760 void *cache;
762 /* We must initialize the internal databases first. */
763 h = iconv_open ("L1", "L1");
764 iconv_close (h);
766 /* See whether we have a cache. */
767 cache = __gconv_get_cache ();
768 if (cache != NULL)
769 /* Yep, use only this information. */
770 insert_cache ();
771 else
773 struct gconv_module *modules;
775 /* No, then use the information read from the gconv-modules file.
776 First add the aliases. */
777 twalk (__gconv_get_alias_db (), insert_print_list);
779 /* Add the from- and to-names from the known modules. */
780 modules = __gconv_get_modules_db ();
781 if (modules != NULL)
782 add_known_names (modules);
785 bool human_readable = isatty (fileno (stdout));
787 if (human_readable)
788 fputs (_("\
789 The following list contain all the coded character sets known. This does\n\
790 not necessarily mean that all combinations of these names can be used for\n\
791 the FROM and TO command line parameters. One coded character set can be\n\
792 listed with several different names (aliases).\n\n "), stdout);
794 /* Now print the collected names. */
795 column = 2;
796 twalk (printlist, human_readable ? do_print_human : do_print);
798 if (human_readable && column != 0)
799 puts ("");