iconv/iconv_prog.c

   1 /* Convert text in given files from the specified from-set to the to-set.
   2    Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, write to the Free
  18    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  19    02111-1307 USA.  */
  20
  21 #include <argp.h>
  22 #include <assert.h>
  23 #include <ctype.h>
  24 #include <errno.h>
  25 #include <error.h>
  26 #include <fcntl.h>
  27 #include <iconv.h>
  28 #include <langinfo.h>
  29 #include <locale.h>
  30 #include <search.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <unistd.h>
  35 #include <libintl.h>
  36 #ifdef _POSIX_MAPPED_FILES
  37 # include <sys/mman.h>
  38 #endif
  39 #include <charmap.h>
  40 #include <gconv_int.h>
  41 #include "iconv_prog.h"
  42
  43 /* Get libc version number.  */
  44 #include "../version.h"
  45
  46 #define PACKAGE _libc_intl_domainname
  47
  48
  49 /* Name and version of program.  */
  50 static void print_version (FILE *stream, struct argp_state *state);
  51 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  52
  53 #define OPT_VERBOSE     1000
  54 #define OPT_LIST        'l'
  55
  56 /* Definitions of arguments for argp functions.  */
  57 static const struct argp_option options[] =
  58 {
  59   { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
  60   { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
  61   { "to-code", 't', "NAME", 0, N_("encoding for output") },
  62   { NULL, 0, NULL, 0, N_("Information:") },
  63   { "list", 'l', NULL, 0, N_("list all known coded character sets") },
  64   { NULL, 0, NULL, 0, N_("Output control:") },
  65   { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
  66   { "output", 'o', "FILE", 0, N_("output file") },
  67   { "silent", 's', NULL, 0, N_("suppress warnings") },
  68   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
  69   { NULL, 0, NULL, 0, NULL }
  70 };
  71
  72 /* Short description of program.  */
  73 static const char doc[] = N_("\
  74 Convert encoding of given files from one encoding to another.");
  75
  76 /* Strings for arguments in help texts.  */
  77 static const char args_doc[] = N_("[FILE...]");
  78
  79 /* Prototype for option handler.  */
  80 static error_t parse_opt (int key, char *arg, struct argp_state *state);
  81
  82 /* Function to print some extra text in the help message.  */
  83 static char *more_help (int key, const char *text, void *input);
  84
  85 /* Data structure to communicate with argp functions.  */
  86 static struct argp argp =
  87 {
  88   options, parse_opt, args_doc, doc, NULL, more_help
  89 };
  90
  91 /* Code sets to convert from and to respectively.  */
  92 static const char *from_code;
  93 static const char *to_code;
  94
  95 /* File to write output to.  If NULL write to stdout.  */
  96 static const char *output_file;
  97
  98 /* Nonzero if verbose ouput is wanted.  */
  99 int verbose;
 100
 101 /* Nonzero if list of all coded character sets is wanted.  */
 102 static int list;
 103
 104 /* If nonzero omit invalid character from output.  */
 105 int omit_invalid;
 106
 107 /* Prototypes for the functions doing the actual work.  */
 108 static int process_block (iconv_t cd, char *addr, size_t len, FILE *output);
 109 static int process_fd (iconv_t cd, int fd, FILE *output);
 110 static int process_file (iconv_t cd, FILE *input, FILE *output);
 111 static void print_known_names (void) internal_function;
 112
 113
 114 int
 115 main (int argc, char *argv[])
 116 {
 117   int status = EXIT_SUCCESS;
 118   int remaining;
 119   FILE *output;
 120   iconv_t cd;
 121   const char *orig_to_code;
 122   struct charmap_t *from_charmap = NULL;
 123   struct charmap_t *to_charmap = NULL;
 124
 125   /* Set locale via LC_ALL.  */
 126   setlocale (LC_ALL, "");
 127
 128   /* Set the text message domain.  */
 129   textdomain (_libc_intl_domainname);
 130
 131   /* Parse and process arguments.  */
 132   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 133
 134   /* List all coded character sets if wanted.  */
 135   if (list)
 136     {
 137       print_known_names ();
 138       exit (EXIT_SUCCESS);
 139     }
 140   if (from_code == NULL)
 141     {
 142       /* The Unix standard says that in this case the charset of the current
 143          locale is used.  */
 144       from_code = nl_langinfo (CODESET);
 145       assert (from_code != NULL);
 146     }
 147   if (to_code == NULL)
 148     {
 149       /* The Unix standard says that in this case the charset of the current
 150          locale is used.  */
 151       to_code = nl_langinfo (CODESET);
 152       assert (to_code != NULL);
 153     }
 154
 155   /* If we have to ignore errors make sure we use the appropriate name for
 156      the to-character-set.  */
 157   orig_to_code = to_code;
 158   if (omit_invalid)
 159     {
 160       const char *errhand = strchrnul (to_code, '/');
 161       int nslash = 2;
 162       char *newp;
 163       char *cp;
 164
 165       if (*errhand == '/')
 166         {
 167           --nslash;
 168           errhand = strchrnul (errhand, '/');
 169
 170           if (*errhand == '/')
 171             {
 172               --nslash;
 173               ++errhand;
 174             }
 175         }
 176
 177       newp = (char *) alloca (errhand - to_code + nslash + 6 + 1);
 178       cp = mempcpy (newp, to_code, errhand - to_code);
 179       while (nslash-- > 0)
 180         *cp++ = '/';
 181       memcpy (cp, "IGNORE", sizeof ("IGNORE"));
 182
 183       to_code = newp;
 184     }
 185
 186   /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
 187      can be file names of charmaps.  In this case iconv will have to read
 188      those charmaps and use them to do the conversion.  But there are
 189      holes in the specification.  There is nothing said that if -f is a
 190      charmap filename that -t must be, too.  And vice versa.  There is
 191      also no word about the symbolic names used.  What if they don't
 192      match?  */
 193   if (strchr (from_code, '/') != NULL)
 194     /* The from-name might be a charmap file name.  Try reading the
 195        file.  */
 196     from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0);
 197
 198   if (strchr (orig_to_code, '/') != NULL)
 199     /* The to-name might be a charmap file name.  Try reading the
 200        file.  */
 201     to_charmap = charmap_read (orig_to_code, /*0, 1,*/1,0, 0);
 202
 203
 204   /* Determine output file.  */
 205   if (output_file != NULL && strcmp (output_file, "-") != 0)
 206     {
 207       output = fopen (output_file, "w");
 208       if (output == NULL)
 209         error (EXIT_FAILURE, errno, _("cannot open output file"));
 210     }
 211   else
 212     output = stdout;
 213
 214   /* At this point we have to handle two cases.  The first one is
 215      where a charmap is used for the from- or to-charset, or both.  We
 216      handle this special since it is very different from the sane way of
 217      doing things.  The other case allows converting using the iconv()
 218      function.  */
 219   if (from_charmap != NULL || to_charmap != NULL)
 220     /* Construct the conversion table and do the conversion.  */
 221     status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
 222                                  argc, remaining, argv, output);
 223   else
 224     {
 225       /* Let's see whether we have these coded character sets.  */
 226       cd = iconv_open (to_code, from_code);
 227       if (cd == (iconv_t) -1)
 228         {
 229           if (errno == EINVAL)
 230             error (EXIT_FAILURE, 0,
 231                    _("conversion from `%s' to `%s' not supported"),
 232                    from_code, orig_to_code);
 233           else
 234             error (EXIT_FAILURE, errno,
 235                    _("failed to start conversion processing"));
 236         }
 237
 238       /* Now process the remaining files.  Write them to stdout or the file
 239          specified with the `-o' parameter.  If we have no file given as
 240          the parameter process all from stdin.  */
 241       if (remaining == argc)
 242         {
 243           if (process_file (cd, stdin, output) != 0)
 244             status = EXIT_FAILURE;
 245         }
 246       else
 247         do
 248           {
 249 #ifdef _POSIX_MAPPED_FILES
 250             struct stat st;
 251             char *addr;
 252 #endif
 253             int fd;
 254
 255             if (verbose)
 256               printf ("%s:\n", argv[remaining]);
 257             if (strcmp (argv[remaining], "-") == 0)
 258               fd = 0;
 259             else
 260               {
 261                 fd = open (argv[remaining], O_RDONLY);
 262
 263                 if (fd == -1)
 264                   {
 265                     error (0, errno, _("cannot open input file `%s'"),
 266                            argv[remaining]);
 267                     status = EXIT_FAILURE;
 268                     continue;
 269                   }
 270               }
 271
 272 #ifdef _POSIX_MAPPED_FILES
 273             /* We have possibilities for reading the input file.  First try
 274                to mmap() it since this will provide the fastest solution.  */
 275             if (fstat (fd, &st) == 0
 276                 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 277                                   fd, 0)) != MAP_FAILED))
 278               {
 279                 /* Yes, we can use mmap().  The descriptor is not needed
 280                    anymore.  */
 281                 if (close (fd) != 0)
 282                   error (EXIT_FAILURE, errno,
 283                          _("error while closing input `%s'"),
 284                          argv[remaining]);
 285
 286                 if (process_block (cd, addr, st.st_size, output) < 0)
 287                   {
 288                     /* Something went wrong.  */
 289                     status = EXIT_FAILURE;
 290
 291                     /* We don't need the input data anymore.  */
 292                     munmap ((void *) addr, st.st_size);
 293
 294                     /* We cannot go on with producing output since it might
 295                        lead to problem because the last output might leave
 296                        the output stream in an undefined state.  */
 297                     break;
 298                   }
 299
 300                 /* We don't need the input data anymore.  */
 301                 munmap ((void *) addr, st.st_size);
 302               }
 303             else
 304 #endif  /* _POSIX_MAPPED_FILES */
 305               {
 306                 /* Read the file in pieces.  */
 307                 if (process_fd (cd, fd, output) != 0)
 308                   {
 309                     /* Something went wrong.  */
 310                     status = EXIT_FAILURE;
 311
 312                     /* We don't need the input file anymore.  */
 313                     close (fd);
 314
 315                     /* We cannot go on with producing output since it might
 316                        lead to problem because the last output might leave
 317                        the output stream in an undefined state.  */
 318                     break;
 319                   }
 320
 321                 /* Now close the file.  */
 322                 close (fd);
 323               }
 324           }
 325         while (++remaining < argc);
 326     }
 327
 328   /* Close the output file now.  */
 329   if (fclose (output))
 330     error (EXIT_FAILURE, errno, _("error while closing output file"));
 331
 332   return status;
 333 }
 334
 335
 336 /* Handle program arguments.  */
 337 static error_t
 338 parse_opt (int key, char *arg, struct argp_state *state)
 339 {
 340   switch (key)
 341     {
 342     case 'f':
 343       from_code = arg;
 344       break;
 345     case 't':
 346       to_code = arg;
 347       break;
 348     case 'o':
 349       output_file = arg;
 350       break;
 351     case 's':
 352       /* Nothing, for now at least.  We are not giving out any information
 353          about missing character or so.  */
 354       break;
 355     case 'c':
 356       /* Omit invalid characters from output.  */
 357       omit_invalid = 1;
 358       break;
 359     case OPT_VERBOSE:
 360       verbose = 1;
 361       break;
 362     case OPT_LIST:
 363       list = 1;
 364       break;
 365     default:
 366       return ARGP_ERR_UNKNOWN;
 367     }
 368   return 0;
 369 }
 370
 371
 372 static char *
 373 more_help (int key, const char *text, void *input)
 374 {
 375   switch (key)
 376     {
 377     case ARGP_KEY_HELP_EXTRA:
 378       /* We print some extra information.  */
 379       return strdup (gettext ("\
 380 Report bugs using the `glibcbug' script to <bugs@gnu.org>.\n"));
 381     default:
 382       break;
 383     }
 384   return (char *) text;
 385 }
 386
 387
 388 /* Print the version information.  */
 389 static void
 390 print_version (FILE *stream, struct argp_state *state)
 391 {
 392   fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
 393   fprintf (stream, gettext ("\
 394 Copyright (C) %s Free Software Foundation, Inc.\n\
 395 This is free software; see the source for copying conditions.  There is NO\n\
 396 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 397 "), "2001");
 398   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 399 }
 400
 401
 402 static int
 403 process_block (iconv_t cd, char *addr, size_t len, FILE *output)
 404 {
 405 #define OUTBUF_SIZE     32768
 406   const char *start = addr;
 407   char outbuf[OUTBUF_SIZE];
 408   char *outptr;
 409   size_t outlen;
 410   size_t n;
 411
 412   while (len > 0)
 413     {
 414       outptr = outbuf;
 415       outlen = OUTBUF_SIZE;
 416       n = iconv (cd, &addr, &len, &outptr, &outlen);
 417
 418       if (outptr != outbuf)
 419         {
 420           /* We have something to write out.  */
 421           int errno_save = errno;
 422
 423           if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 424               || ferror (output))
 425             {
 426               /* Error occurred while printing the result.  */
 427               error (0, 0, _("\
 428 conversion stopped due to problem in writing the output"));
 429               return -1;
 430             }
 431
 432           errno = errno_save;
 433         }
 434
 435       if (n != (size_t) -1)
 436         {
 437           /* All the input test is processed.  For state-dependent
 438              character sets we have to flush the state now.  */
 439           outptr = outbuf;
 440           outlen = OUTBUF_SIZE;
 441           (void) iconv (cd, NULL, NULL, &outptr, &outlen);
 442
 443           if (outptr != outbuf)
 444             {
 445               /* We have something to write out.  */
 446               int errno_save = errno;
 447
 448               if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 449                   || ferror (output))
 450                 {
 451                   /* Error occurred while printing the result.  */
 452                   error (0, 0, _("\
 453 conversion stopped due to problem in writing the output"));
 454                   return -1;
 455                 }
 456
 457               errno = errno_save;
 458             }
 459
 460           break;
 461         }
 462
 463       if (errno != E2BIG)
 464         {
 465           /* iconv() ran into a problem.  */
 466           switch (errno)
 467             {
 468             case EILSEQ:
 469               error (0, 0, _("illegal input sequence at position %ld"),
 470                      (long) (addr - start));
 471               break;
 472             case EINVAL:
 473               error (0, 0, _("\
 474 incomplete character or shift sequence at end of buffer"));
 475               break;
 476             case EBADF:
 477               error (0, 0, _("internal error (illegal descriptor)"));
 478               break;
 479             default:
 480               error (0, 0, _("unknown iconv() error %d"), errno);
 481               break;
 482             }
 483
 484           return -1;
 485         }
 486     }
 487
 488   return 0;
 489 }
 490
 491
 492 static int
 493 process_fd (iconv_t cd, int fd, FILE *output)
 494 {
 495   /* we have a problem with reading from a desriptor since we must not
 496      provide the iconv() function an incomplete character or shift
 497      sequence at the end of the buffer.  Since we have to deal with
 498      arbitrary encodings we must read the whole text in a buffer and
 499      process it in one step.  */
 500   static char *inbuf = NULL;
 501   static size_t maxlen = 0;
 502   char *inptr = NULL;
 503   size_t actlen = 0;
 504
 505   while (actlen < maxlen)
 506     {
 507       ssize_t n = read (fd, inptr, maxlen - actlen);
 508
 509       if (n == 0)
 510         /* No more text to read.  */
 511         break;
 512
 513       if (n == -1)
 514         {
 515           /* Error while reading.  */
 516           error (0, errno, _("error while reading the input"));
 517           return -1;
 518         }
 519
 520       inptr += n;
 521       actlen += n;
 522     }
 523
 524   if (actlen == maxlen)
 525     while (1)
 526       {
 527         ssize_t n;
 528
 529         /* Increase the buffer.  */
 530         maxlen += 32768;
 531         inbuf = realloc (inbuf, maxlen);
 532         if (inbuf == NULL)
 533           error (0, errno, _("unable to allocate buffer for input"));
 534         inptr = inbuf + actlen;
 535
 536         do
 537           {
 538             n = read (fd, inptr, maxlen - actlen);
 539
 540             if (n == 0)
 541               /* No more text to read.  */
 542               break;
 543
 544             if (n == -1)
 545               {
 546                 /* Error while reading.  */
 547                 error (0, errno, _("error while reading the input"));
 548                 return -1;
 549               }
 550
 551             inptr += n;
 552             actlen += n;
 553           }
 554         while (actlen < maxlen);
 555
 556         if (n == 0)
 557           /* Break again so we leave both loops.  */
 558           break;
 559       }
 560
 561   /* Now we have all the input in the buffer.  Process it in one run.  */
 562   return process_block (cd, inbuf, actlen, output);
 563 }
 564
 565
 566 static int
 567 process_file (iconv_t cd, FILE *input, FILE *output)
 568 {
 569   /* This should be safe since we use this function only for `stdin' and
 570      we haven't read anything so far.  */
 571   return process_fd (cd, fileno (input), output);
 572 }
 573
 574
 575 /* Print all known character sets/encodings.  */
 576 static void *printlist;
 577 static size_t column;
 578 static int not_first;
 579
 580 static void
 581 insert_print_list (const void *nodep, VISIT value, int level)
 582 {
 583   if (value == leaf || value == postorder)
 584     {
 585       const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
 586       tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
 587     }
 588 }
 589
 590 static void
 591 do_print_human  (const void *nodep, VISIT value, int level)
 592 {
 593   if (value == leaf || value == postorder)
 594     {
 595       const char *s = *(const char **) nodep;
 596       size_t len = strlen (s);
 597       size_t cnt;
 598
 599       while (len > 0 && s[len - 1] == '/')
 600         --len;
 601
 602       for (cnt = 0; cnt < len; ++cnt)
 603         if (isalnum (s[cnt]))
 604           break;
 605       if (cnt == len)
 606         return;
 607
 608       if (not_first)
 609         {
 610           putchar (',');
 611           ++column;
 612
 613           if (column > 2 && column + len > 77)
 614             {
 615               fputs ("\n  ", stdout);
 616               column = 2;
 617             }
 618           else
 619             {
 620               putchar (' ');
 621               ++column;
 622             }
 623         }
 624       else
 625         not_first = 1;
 626
 627       fwrite (s, len, 1, stdout);
 628       column += len;
 629     }
 630 }
 631
 632 static void
 633 do_print  (const void *nodep, VISIT value, int level)
 634 {
 635   if (value == leaf || value == postorder)
 636     {
 637       const char *s = *(const char **) nodep;
 638
 639       puts (s);
 640     }
 641 }
 642
 643 static void
 644 internal_function
 645 add_known_names (struct gconv_module *node)
 646 {
 647   if (node->left != NULL)
 648     add_known_names (node->left);
 649   if (node->right != NULL)
 650     add_known_names (node->right);
 651   do
 652     {
 653       if (strcmp (node->from_string, "INTERNAL"))
 654         tsearch (node->from_string, &printlist,
 655                  (__compar_fn_t) strverscmp);
 656       if (strcmp (node->to_string, "INTERNAL"))
 657         tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
 658
 659       node = node->same;
 660     }
 661   while (node != NULL);
 662 }
 663
 664 static void
 665 internal_function
 666 print_known_names (void)
 667 {
 668   iconv_t h;
 669
 670   /* We must initialize the internal databases first.  */
 671   h = iconv_open ("L1", "L1");
 672   iconv_close (h);
 673
 674   /* First add the aliases.  */
 675   twalk (__gconv_alias_db, insert_print_list);
 676
 677   /* Add the from- and to-names from the known modules.  */
 678   add_known_names (__gconv_modules_db);
 679
 680   fputs (_("\
 681 The following list contain all the coded character sets known.  This does\n\
 682 not necessarily mean that all combinations of these names can be used for\n\
 683 the FROM and TO command line parameters.  One coded character set can be\n\
 684 listed with several different names (aliases).\n\n  "), stdout);
 685
 686   /* Now print the collected names.  */
 687   column = 2;
 688   if (isatty (fileno (stdout)))
 689     {
 690       twalk (printlist, do_print_human);
 691
 692       if (column != 0)
 693         puts ("");
 694     }
 695   else
 696     twalk (printlist, do_print);
 697 }