1 /* Convert text in given files from the specified from-set to the to-set.
2 Copyright (C) 1998-2011, 2012 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software Foundation,
18 Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
36 #ifdef _POSIX_MAPPED_FILES
37 # include <sys/mman.h>
40 #include <gconv_int.h>
41 #include "iconv_prog.h"
42 #include "iconvconfig.h"
44 /* Get libc version number. */
45 #include "../version.h"
47 #define PACKAGE _libc_intl_domainname
50 /* Name and version of program. */
51 static void print_version (FILE *stream
, struct argp_state
*state
);
52 void (*argp_program_version_hook
) (FILE *, struct argp_state
*) = print_version
;
54 #define OPT_VERBOSE 1000
57 /* Definitions of arguments for argp functions. */
58 static const struct argp_option options
[] =
60 { NULL
, 0, NULL
, 0, N_("Input/Output format specification:") },
61 { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
62 { "to-code", 't', "NAME", 0, N_("encoding for output") },
63 { NULL
, 0, NULL
, 0, N_("Information:") },
64 { "list", 'l', NULL
, 0, N_("list all known coded character sets") },
65 { NULL
, 0, NULL
, 0, N_("Output control:") },
66 { NULL
, 'c', NULL
, 0, N_("omit invalid characters from output") },
67 { "output", 'o', "FILE", 0, N_("output file") },
68 { "silent", 's', NULL
, 0, N_("suppress warnings") },
69 { "verbose", OPT_VERBOSE
, NULL
, 0, N_("print progress information") },
70 { NULL
, 0, NULL
, 0, NULL
}
73 /* Short description of program. */
74 static const char doc
[] = N_("\
75 Convert encoding of given files from one encoding to another.");
77 /* Strings for arguments in help texts. */
78 static const char args_doc
[] = N_("[FILE...]");
80 /* Prototype for option handler. */
81 static error_t
parse_opt (int key
, char *arg
, struct argp_state
*state
);
83 /* Function to print some extra text in the help message. */
84 static char *more_help (int key
, const char *text
, void *input
);
86 /* Data structure to communicate with argp functions. */
87 static struct argp argp
=
89 options
, parse_opt
, args_doc
, doc
, NULL
, more_help
92 /* Code sets to convert from and to respectively. An empty string as the
93 default causes the 'iconv_open' function to look up the charset of the
94 currently selected locale and use it. */
95 static const char *from_code
= "";
96 static const char *to_code
= "";
98 /* File to write output to. If NULL write to stdout. */
99 static const char *output_file
;
101 /* Nonzero if verbose ouput is wanted. */
104 /* Nonzero if list of all coded character sets is wanted. */
107 /* If nonzero omit invalid character from output. */
110 /* Prototypes for the functions doing the actual work. */
111 static int process_block (iconv_t cd
, char *addr
, size_t len
, FILE **output
,
112 const char *output_file
);
113 static int process_fd (iconv_t cd
, int fd
, FILE **output
,
114 const char *output_file
);
115 static int process_file (iconv_t cd
, FILE *input
, FILE **output
,
116 const char *output_file
);
117 static void print_known_names (void) internal_function
;
121 main (int argc
, char *argv
[])
123 int status
= EXIT_SUCCESS
;
126 const char *orig_to_code
;
127 struct charmap_t
*from_charmap
= NULL
;
128 struct charmap_t
*to_charmap
= NULL
;
130 /* Set locale via LC_ALL. */
131 setlocale (LC_ALL
, "");
133 /* Set the text message domain. */
134 textdomain (_libc_intl_domainname
);
136 /* Parse and process arguments. */
137 argp_parse (&argp
, argc
, argv
, 0, &remaining
, NULL
);
139 /* List all coded character sets if wanted. */
142 print_known_names ();
146 /* If we have to ignore errors make sure we use the appropriate name for
147 the to-character-set. */
148 orig_to_code
= to_code
;
151 const char *errhand
= strchrnul (to_code
, '/');
159 errhand
= strchrnul (errhand
, '/');
164 errhand
= strchr (errhand
, '\0');
168 newp
= (char *) alloca (errhand
- to_code
+ nslash
+ 7 + 1);
169 cp
= mempcpy (newp
, to_code
, errhand
- to_code
);
174 memcpy (cp
, "IGNORE", sizeof ("IGNORE"));
179 /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
180 can be file names of charmaps. In this case iconv will have to read
181 those charmaps and use them to do the conversion. But there are
182 holes in the specification. There is nothing said that if -f is a
183 charmap filename that -t must be, too. And vice versa. There is
184 also no word about the symbolic names used. What if they don't
186 if (strchr (from_code
, '/') != NULL
)
187 /* The from-name might be a charmap file name. Try reading the
189 from_charmap
= charmap_read (from_code
, /*0, 1*/1, 0, 0, 0);
191 if (strchr (orig_to_code
, '/') != NULL
)
192 /* The to-name might be a charmap file name. Try reading the
194 to_charmap
= charmap_read (orig_to_code
, /*0, 1,*/1, 0, 0, 0);
197 /* At this point we have to handle two cases. The first one is
198 where a charmap is used for the from- or to-charset, or both. We
199 handle this special since it is very different from the sane way of
200 doing things. The other case allows converting using the iconv()
202 if (from_charmap
!= NULL
|| to_charmap
!= NULL
)
203 /* Construct the conversion table and do the conversion. */
204 status
= charmap_conversion (from_code
, from_charmap
, to_code
, to_charmap
,
205 argc
, remaining
, argv
, output_file
);
208 /* Let's see whether we have these coded character sets. */
209 cd
= iconv_open (to_code
, from_code
);
210 if (cd
== (iconv_t
) -1)
214 /* Try to be nice with the user and tell her which of the
215 two encoding names is wrong. This is possible because
216 all supported encodings can be converted from/to Unicode,
217 in other words, because the graph of encodings is
220 (iconv_open ("UTF-8", from_code
) == (iconv_t
) -1
223 (iconv_open (to_code
, "UTF-8") == (iconv_t
) -1
225 const char *from_pretty
=
226 (from_code
[0] ? from_code
: nl_langinfo (CODESET
));
227 const char *to_pretty
=
228 (orig_to_code
[0] ? orig_to_code
: nl_langinfo (CODESET
));
235 conversions from `%s' and to `%s' are not supported"),
236 from_pretty
, to_pretty
);
239 _("conversion from `%s' is not supported"),
246 _("conversion to `%s' is not supported"),
250 _("conversion from `%s' to `%s' is not supported"),
251 from_pretty
, to_pretty
);
254 argp_help (&argp
, stderr
, ARGP_HELP_SEE
,
255 program_invocation_short_name
);
259 error (EXIT_FAILURE
, errno
,
260 _("failed to start conversion processing"));
263 /* The output file. Will be opened when we are ready to produce
267 /* Now process the remaining files. Write them to stdout or the file
268 specified with the `-o' parameter. If we have no file given as
269 the parameter process all from stdin. */
270 if (remaining
== argc
)
272 if (process_file (cd
, stdin
, &output
, output_file
) != 0)
273 status
= EXIT_FAILURE
;
278 #ifdef _POSIX_MAPPED_FILES
285 fprintf (stderr
, "%s:\n", argv
[remaining
]);
286 if (strcmp (argv
[remaining
], "-") == 0)
290 fd
= open (argv
[remaining
], O_RDONLY
);
294 error (0, errno
, _("cannot open input file `%s'"),
296 status
= EXIT_FAILURE
;
301 #ifdef _POSIX_MAPPED_FILES
302 /* We have possibilities for reading the input file. First try
303 to mmap() it since this will provide the fastest solution. */
304 if (fstat (fd
, &st
) == 0
305 && ((addr
= mmap (NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
,
306 fd
, 0)) != MAP_FAILED
))
308 /* Yes, we can use mmap(). The descriptor is not needed
311 error (EXIT_FAILURE
, errno
,
312 _("error while closing input `%s'"),
315 ret
= process_block (cd
, addr
, st
.st_size
, &output
,
318 /* We don't need the input data anymore. */
319 munmap ((void *) addr
, st
.st_size
);
323 status
= EXIT_FAILURE
;
326 /* We cannot go on with producing output since it might
327 lead to problem because the last output might leave
328 the output stream in an undefined state. */
333 #endif /* _POSIX_MAPPED_FILES */
335 /* Read the file in pieces. */
336 ret
= process_fd (cd
, fd
, &output
, output_file
);
338 /* Now close the file. */
343 /* Something went wrong. */
344 status
= EXIT_FAILURE
;
347 /* We cannot go on with producing output since it might
348 lead to problem because the last output might leave
349 the output stream in an undefined state. */
354 while (++remaining
< argc
);
356 /* Close the output file now. */
357 if (output
!= NULL
&& fclose (output
))
358 error (EXIT_FAILURE
, errno
, _("error while closing output file"));
365 /* Handle program arguments. */
367 parse_opt (int key
, char *arg
, struct argp_state
*state
)
381 /* Nothing, for now at least. We are not giving out any information
382 about missing character or so. */
385 /* Omit invalid characters from output. */
395 return ARGP_ERR_UNKNOWN
;
402 more_help (int key
, const char *text
, void *input
)
406 case ARGP_KEY_HELP_EXTRA
:
407 /* We print some extra information. */
408 return strdup (gettext ("\
409 For bug reporting instructions, please see:\n\
410 <http://www.gnu.org/software/libc/bugs.html>.\n"));
414 return (char *) text
;
418 /* Print the version information. */
420 print_version (FILE *stream
, struct argp_state
*state
)
422 fprintf (stream
, "iconv (GNU %s) %s\n", PACKAGE
, VERSION
);
423 fprintf (stream
, gettext ("\
424 Copyright (C) %s Free Software Foundation, Inc.\n\
425 This is free software; see the source for copying conditions. There is NO\n\
426 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
428 fprintf (stream
, gettext ("Written by %s.\n"), "Ulrich Drepper");
433 write_output (const char *outbuf
, const char *outptr
, FILE **output
,
434 const char *output_file
)
436 /* We have something to write out. */
437 int errno_save
= errno
;
441 /* Determine output file. */
442 if (output_file
!= NULL
&& strcmp (output_file
, "-") != 0)
444 *output
= fopen (output_file
, "w");
446 error (EXIT_FAILURE
, errno
, _("cannot open output file"));
452 if (fwrite (outbuf
, 1, outptr
- outbuf
, *output
) < (size_t) (outptr
- outbuf
)
455 /* Error occurred while printing the result. */
457 conversion stopped due to problem in writing the output"));
468 process_block (iconv_t cd
, char *addr
, size_t len
, FILE **output
,
469 const char *output_file
)
471 #define OUTBUF_SIZE 32768
472 const char *start
= addr
;
473 char outbuf
[OUTBUF_SIZE
];
482 outlen
= OUTBUF_SIZE
;
483 n
= iconv (cd
, &addr
, &len
, &outptr
, &outlen
);
485 if (n
== (size_t) -1 && omit_invalid
&& errno
== EILSEQ
)
494 if (outptr
!= outbuf
)
496 ret
= write_output (outbuf
, outptr
, output
, output_file
);
501 if (n
!= (size_t) -1)
503 /* All the input test is processed. For state-dependent
504 character sets we have to flush the state now. */
506 outlen
= OUTBUF_SIZE
;
507 n
= iconv (cd
, NULL
, NULL
, &outptr
, &outlen
);
509 if (outptr
!= outbuf
)
511 ret
= write_output (outbuf
, outptr
, output
, output_file
);
516 if (n
!= (size_t) -1)
519 if (omit_invalid
&& errno
== EILSEQ
)
528 /* iconv() ran into a problem. */
533 error (0, 0, _("illegal input sequence at position %ld"),
534 (long int) (addr
- start
));
538 incomplete character or shift sequence at end of buffer"));
541 error (0, 0, _("internal error (illegal descriptor)"));
544 error (0, 0, _("unknown iconv() error %d"), errno
);
557 process_fd (iconv_t cd
, int fd
, FILE **output
, const char *output_file
)
559 /* we have a problem with reading from a desriptor since we must not
560 provide the iconv() function an incomplete character or shift
561 sequence at the end of the buffer. Since we have to deal with
562 arbitrary encodings we must read the whole text in a buffer and
563 process it in one step. */
564 static char *inbuf
= NULL
;
565 static size_t maxlen
= 0;
569 while (actlen
< maxlen
)
571 ssize_t n
= read (fd
, inptr
, maxlen
- actlen
);
574 /* No more text to read. */
579 /* Error while reading. */
580 error (0, errno
, _("error while reading the input"));
588 if (actlen
== maxlen
)
594 /* Increase the buffer. */
595 new_inbuf
= (char *) realloc (inbuf
, maxlen
+ 32768);
596 if (new_inbuf
== NULL
)
598 error (0, errno
, _("unable to allocate buffer for input"));
603 inptr
= inbuf
+ actlen
;
607 n
= read (fd
, inptr
, maxlen
- actlen
);
610 /* No more text to read. */
615 /* Error while reading. */
616 error (0, errno
, _("error while reading the input"));
623 while (actlen
< maxlen
);
626 /* Break again so we leave both loops. */
630 /* Now we have all the input in the buffer. Process it in one run. */
631 return process_block (cd
, inbuf
, actlen
, output
, output_file
);
636 process_file (iconv_t cd
, FILE *input
, FILE **output
, const char *output_file
)
638 /* This should be safe since we use this function only for `stdin' and
639 we haven't read anything so far. */
640 return process_fd (cd
, fileno (input
), output
, output_file
);
644 /* Print all known character sets/encodings. */
645 static void *printlist
;
646 static size_t column
;
647 static int not_first
;
650 insert_print_list (const void *nodep
, VISIT value
, int level
)
652 if (value
== leaf
|| value
== postorder
)
654 const struct gconv_alias
*s
= *(const struct gconv_alias
**) nodep
;
655 tsearch (s
->fromname
, &printlist
, (__compar_fn_t
) strverscmp
);
660 do_print_human (const void *nodep
, VISIT value
, int level
)
662 if (value
== leaf
|| value
== postorder
)
664 const char *s
= *(const char **) nodep
;
665 size_t len
= strlen (s
);
668 while (len
> 0 && s
[len
- 1] == '/')
671 for (cnt
= 0; cnt
< len
; ++cnt
)
672 if (isalnum (s
[cnt
]))
682 if (column
> 2 && column
+ len
> 77)
684 fputs ("\n ", stdout
);
696 fwrite (s
, len
, 1, stdout
);
702 do_print (const void *nodep
, VISIT value
, int level
)
704 if (value
== leaf
|| value
== postorder
)
706 const char *s
= *(const char **) nodep
;
714 add_known_names (struct gconv_module
*node
)
716 if (node
->left
!= NULL
)
717 add_known_names (node
->left
);
718 if (node
->right
!= NULL
)
719 add_known_names (node
->right
);
722 if (strcmp (node
->from_string
, "INTERNAL") != 0)
723 tsearch (node
->from_string
, &printlist
, (__compar_fn_t
) strverscmp
);
724 if (strcmp (node
->to_string
, "INTERNAL") != 0)
725 tsearch (node
->to_string
, &printlist
, (__compar_fn_t
) strverscmp
);
729 while (node
!= NULL
);
736 const struct gconvcache_header
*header
;
738 const struct hash_entry
*hashtab
;
741 header
= (const struct gconvcache_header
*) __gconv_get_cache ();
742 strtab
= (char *) header
+ header
->string_offset
;
743 hashtab
= (struct hash_entry
*) ((char *) header
+ header
->hash_offset
);
745 for (cnt
= 0; cnt
< header
->hash_size
; ++cnt
)
746 if (hashtab
[cnt
].string_offset
!= 0)
748 const char *str
= strtab
+ hashtab
[cnt
].string_offset
;
750 if (strcmp (str
, "INTERNAL") != 0)
751 tsearch (str
, &printlist
, (__compar_fn_t
) strverscmp
);
758 print_known_names (void)
763 /* We must initialize the internal databases first. */
764 h
= iconv_open ("L1", "L1");
767 /* See whether we have a cache. */
768 cache
= __gconv_get_cache ();
770 /* Yep, use only this information. */
774 struct gconv_module
*modules
;
776 /* No, then use the information read from the gconv-modules file.
777 First add the aliases. */
778 twalk (__gconv_get_alias_db (), insert_print_list
);
780 /* Add the from- and to-names from the known modules. */
781 modules
= __gconv_get_modules_db ();
783 add_known_names (modules
);
786 bool human_readable
= isatty (fileno (stdout
));
790 The following list contain all the coded character sets known. This does\n\
791 not necessarily mean that all combinations of these names can be used for\n\
792 the FROM and TO command line parameters. One coded character set can be\n\
793 listed with several different names (aliases).\n\n "), stdout
);
795 /* Now print the collected names. */
797 twalk (printlist
, human_readable
? do_print_human
: do_print
);
799 if (human_readable
&& column
!= 0)