iconv/iconv_prog.c

   1 /* Convert text in given files from the specified from-set to the to-set.
   2    Copyright (C) 1998, 1999 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Library General Public License as
   8    published by the Free Software Foundation; either version 2 of the
   9    License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Library General Public License for more details.
  15
  16    You should have received a copy of the GNU Library General Public
  17    License along with the GNU C Library; see the file COPYING.LIB.  If not,
  18    write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19    Boston, MA 02111-1307, USA.  */
  20
  21 #include <argp.h>
  22 #include <ctype.h>
  23 #include <errno.h>
  24 #include <error.h>
  25 #include <fcntl.h>
  26 #include <iconv.h>
  27 #include <locale.h>
  28 #include <search.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #ifdef _POSIX_MAPPED_FILES
  34 # include <sys/mman.h>
  35 #endif
  36 #include <gconv_int.h>
  37
  38 /* Get libc version number.  */
  39 #include "../version.h"
  40
  41 #define PACKAGE _libc_intl_domainname
  42
  43
  44 /* Name and version of program.  */
  45 static void print_version (FILE *stream, struct argp_state *state);
  46 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  47
  48 #define OPT_VERBOSE     1000
  49 #define OPT_LIST        1001
  50
  51 /* Definitions of arguments for argp functions.  */
  52 static const struct argp_option options[] =
  53 {
  54   { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
  55   { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
  56   { "to-code", 't', "NAME", 0, N_("encoding for output") },
  57   { NULL, 0, NULL, 0, N_("Information:") },
  58   { "list", OPT_LIST, NULL, 0, N_("list all known coded character sets") },
  59   { NULL, 0, NULL, 0, N_("Output control:") },
  60   { "output", 'o', "FILE", 0, N_("output file") },
  61   { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
  62   { NULL, 0, NULL, 0, NULL }
  63 };
  64
  65 /* Short description of program.  */
  66 static const char doc[] = N_("\
  67 Convert encoding of given files from one encoding to another.");
  68
  69 /* Strings for arguments in help texts.  */
  70 static const char args_doc[] = N_("[FILE...]");
  71
  72 /* Prototype for option handler.  */
  73 static error_t parse_opt __P ((int key, char *arg, struct argp_state *state));
  74
  75 /* Function to print some extra text in the help message.  */
  76 static char *more_help __P ((int key, const char *text, void *input));
  77
  78 /* Data structure to communicate with argp functions.  */
  79 static struct argp argp =
  80 {
  81   options, parse_opt, args_doc, doc, NULL, more_help
  82 };
  83
  84 /* Code sets to convert from and to respectively.  */
  85 static const char *from_code;
  86 static const char *to_code;
  87
  88 /* File to write output to.  If NULL write to stdout.  */
  89 static const char *output_file;
  90
  91 /* Nonzero if verbose ouput is wanted.  */
  92 static int verbose;
  93
  94 /* Nonzero if list of all coded character sets is wanted.  */
  95 static int list;
  96
  97 /* Prototypes for the functions doing the actual work.  */
  98 static int process_block (iconv_t cd, const char *addr, size_t len,
  99                           FILE *output);
 100 static int process_fd (iconv_t cd, int fd, FILE *output);
 101 static int process_file (iconv_t cd, FILE *input, FILE *output);
 102 static void print_known_names (void) internal_function;
 103
 104
 105 int
 106 main (int argc, char *argv[])
 107 {
 108   int status = EXIT_SUCCESS;
 109   int remaining;
 110   FILE *output;
 111   iconv_t cd;
 112
 113   /* Set locale via LC_ALL.  */
 114   setlocale (LC_ALL, "");
 115
 116   /* Set the text message domain.  */
 117   textdomain (_libc_intl_domainname);
 118
 119   /* Parse and process arguments.  */
 120   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 121
 122   /* List all coded character sets if wanted.  */
 123   if (list)
 124     {
 125       print_known_names ();
 126       exit (EXIT_SUCCESS);
 127     }
 128
 129   /* If either the from- or to-code is not specified this is an error
 130      since we do not know what to do.  */
 131   if (from_code == NULL && to_code == NULL)
 132     error (EXIT_FAILURE, 0,
 133            _("neither original nor target encoding specified"));
 134   if (from_code == NULL)
 135     error (EXIT_FAILURE, 0, _("original encoding not specified using `-f'"));
 136   if (to_code == NULL)
 137     error (EXIT_FAILURE, 0, _("target encoding not specified using `-t'"));
 138
 139   /* Let's see whether we have these coded character sets.  */
 140   cd = iconv_open (to_code, from_code);
 141   if (cd == (iconv_t) -1)
 142     {
 143       if (errno == EINVAL)
 144         error (EXIT_FAILURE, 0, _("conversion from `%s' to `%s' not supported"),
 145                from_code, to_code);
 146       else
 147         error (EXIT_FAILURE, errno, _("failed to start conversion processing"));
 148     }
 149
 150   /* Determine output file.  */
 151   if (output_file != NULL)
 152     {
 153       output = fopen (output_file, "w");
 154       if (output == NULL)
 155         error (EXIT_FAILURE, errno, _("cannot open output file"));
 156     }
 157   else
 158     output = stdout;
 159
 160   /* Now process the remaining files.  Write them to stdout or the file
 161      specified with the `-o' parameter.  If we have no file given as
 162      the parameter process all from stdin.  */
 163   if (remaining == argc)
 164     process_file (cd, stdin, output);
 165   else
 166     do
 167       {
 168         struct stat st;
 169         const char *addr;
 170         int fd = open (argv[remaining], O_RDONLY);
 171
 172         if (verbose)
 173           printf ("%s:\n", argv[remaining]);
 174
 175         if (fd == -1)
 176           {
 177             error (0, errno, _("cannot open input file `%s'"),
 178                    argv[remaining]);
 179             status = EXIT_FAILURE;
 180             continue;
 181           }
 182
 183 #ifdef _POSIX_MAPPED_FILES
 184         /* We have possibilities for reading the input file.  First try
 185            to mmap() it since this will provide the fastest solution.  */
 186         if (fstat (fd, &st) == 0
 187             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0))
 188                 != MAP_FAILED))
 189           {
 190             /* Yes, we can use mmap().  The descriptor is not needed
 191                anymore.  */
 192             if (close (fd) != 0)
 193               error (EXIT_FAILURE, errno, _("error while closing input `%s'"),
 194                      argv[remaining]);
 195
 196             if (process_block (cd, addr, st.st_size, output) < 0)
 197               {
 198                 /* Something went wrong.  */
 199                 status = EXIT_FAILURE;
 200
 201                 /* We don't need the input data anymore.  */
 202                 munmap ((void *) addr, st.st_size);
 203
 204                 /* We cannot go on with producing output since it might
 205                    lead to problem because the last output might leave
 206                    the output stream in an undefined state.  */
 207                 break;
 208               }
 209
 210             /* We don't need the input data anymore.  */
 211             munmap ((void *) addr, st.st_size);
 212           }
 213         else
 214 #endif  /* _POSIX_MAPPED_FILES */
 215           {
 216             /* Read the file in pieces.  */
 217             if (process_fd (cd, fd, output) != 0)
 218               {
 219                 /* Something went wrong.  */
 220                 status = EXIT_FAILURE;
 221
 222                 /* We don't need the input file anymore.  */
 223                 close (fd);
 224
 225                 /* We cannot go on with producing output since it might
 226                    lead to problem because the last output might leave
 227                    the output stream in an undefined state.  */
 228                 break;
 229               }
 230
 231             /* Now close the file.  */
 232             close (fd);
 233           }
 234       }
 235     while (++remaining < argc);
 236
 237   /* Close the output file now.  */
 238   if (fclose (output))
 239     error (EXIT_FAILURE, errno, _("error while closing output file"));
 240
 241   return status;
 242 }
 243
 244
 245 /* Handle program arguments.  */
 246 static error_t
 247 parse_opt (int key, char *arg, struct argp_state *state)
 248 {
 249   switch (key)
 250     {
 251     case 'f':
 252       from_code = arg;
 253       break;
 254     case 't':
 255       to_code = arg;
 256       break;
 257     case 'o':
 258       output_file = arg;
 259       break;
 260     case OPT_VERBOSE:
 261       verbose = 1;
 262       break;
 263     case OPT_LIST:
 264       list = 1;
 265       break;
 266     default:
 267       return ARGP_ERR_UNKNOWN;
 268     }
 269   return 0;
 270 }
 271
 272
 273 static char *
 274 more_help (int key, const char *text, void *input)
 275 {
 276   switch (key)
 277     {
 278     case ARGP_KEY_HELP_EXTRA:
 279       /* We print some extra information.  */
 280       return strdup (gettext ("\
 281 Report bugs using the `glibcbug' script to <bugs@gnu.org>.\n"));
 282     default:
 283       break;
 284     }
 285   return (char *) text;
 286 }
 287
 288
 289 /* Print the version information.  */
 290 static void
 291 print_version (FILE *stream, struct argp_state *state)
 292 {
 293   fprintf (stream, "iconv (GNU %s) %s\n", PACKAGE, VERSION);
 294   fprintf (stream, gettext ("\
 295 Copyright (C) %s Free Software Foundation, Inc.\n\
 296 This is free software; see the source for copying conditions.  There is NO\n\
 297 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 298 "), "1999");
 299   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 300 }
 301
 302
 303 static int
 304 process_block (iconv_t cd, const char *addr, size_t len, FILE *output)
 305 {
 306 #define OUTBUF_SIZE     32768
 307   const char *start = addr;
 308   char outbuf[OUTBUF_SIZE];
 309   char *outptr;
 310   size_t outlen;
 311   size_t n;
 312
 313   while (len > 0)
 314     {
 315       outptr = outbuf;
 316       outlen = OUTBUF_SIZE;
 317       n = iconv (cd, &addr, &len, &outptr, &outlen);
 318
 319       if (outptr != outbuf)
 320         {
 321           /* We have something to write out.  */
 322           if (fwrite (outbuf, 1, outptr - outbuf, output) < outptr - outbuf
 323               || ferror (output))
 324             {
 325               /* Error occurred while printing the result.  */
 326               error (0, 0, _("\
 327 conversion stopped due to problem in writing the output"));
 328               return -1;
 329             }
 330         }
 331
 332       if (n != (size_t) -1)
 333         /* Everything is processed.  */
 334         break;
 335
 336       if (errno != E2BIG)
 337         {
 338           /* iconv() ran into a problem.  */
 339           switch (errno)
 340             {
 341             case EILSEQ:
 342               error (0, 0, _("illegal input sequence at position %ld"),
 343                      addr - start);
 344               break;
 345             case EINVAL:
 346               error (0, 0, _("\
 347 incomplete character or shift sequence at end of buffer"));
 348               break;
 349             case EBADF:
 350               error (0, 0, _("internal error (illegal descriptor)"));
 351               break;
 352             default:
 353               error (0, 0, _("unknown iconv() error %d"), errno);
 354               break;
 355             }
 356
 357           return -1;
 358         }
 359     }
 360
 361   return 0;
 362 }
 363
 364
 365 static int
 366 process_fd (iconv_t cd, int fd, FILE *output)
 367 {
 368   /* we have a problem with reading from a desriptor since we must not
 369      provide the iconv() function an incomplete character or shift
 370      sequence at the end of the buffer.  Since we have to deal with
 371      arbitrary encodings we must read the whole text in a buffer and
 372      process it in one step.  */
 373   static char *inbuf = NULL;
 374   static size_t maxlen = 0;
 375   char *inptr = NULL;
 376   size_t actlen = 0;
 377
 378   while (actlen < maxlen)
 379     {
 380       size_t n = read (fd, inptr, maxlen - actlen);
 381
 382       if (n == 0)
 383         /* No more text to read.  */
 384         break;
 385
 386       if (n == -1)
 387         {
 388           /* Error while reading.  */
 389           error (0, errno, _("error while reading the input"));
 390           return -1;
 391         }
 392
 393       inptr += n;
 394       actlen += n;
 395     }
 396
 397   if (actlen == maxlen)
 398     while (1)
 399       {
 400         size_t n;
 401
 402         /* Increase the buffer.  */
 403         maxlen += 32768;
 404         inbuf = realloc (inbuf, maxlen);
 405         if (inbuf == NULL)
 406           error (0, errno, _("unable to allocate buffer for input"));
 407         inptr = inbuf + actlen;
 408
 409         do
 410           {
 411             n = read (fd, inptr, maxlen - actlen);
 412
 413             if (n == 0)
 414               /* No more text to read.  */
 415               break;
 416
 417             if (n == -1)
 418               {
 419                 /* Error while reading.  */
 420                 error (0, errno, _("error while reading the input"));
 421                 return -1;
 422               }
 423
 424             inptr += n;
 425             actlen += n;
 426           }
 427         while (actlen < maxlen);
 428
 429         if (n == 0)
 430           /* Break again so we leave both loops.  */
 431           break;
 432       }
 433
 434   /* Now we have all the input in the buffer.  Process it in one run.  */
 435   return process_block (cd, inbuf, actlen, output);
 436 }
 437
 438
 439 static int
 440 process_file (iconv_t cd, FILE *input, FILE *output)
 441 {
 442   /* This should be safe since we use this function only for `stdin' and
 443      we haven't read anything so far.  */
 444   return process_fd (cd, fileno (input), output);
 445 }
 446
 447
 448 /* Print all known character sets/encodings.  */
 449 static void *printlist;
 450 static size_t column;
 451 static int not_first;
 452
 453 static void
 454 insert_print_list (const void *nodep, VISIT value, int level)
 455 {
 456   if (value == leaf || value == postorder)
 457     {
 458       const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
 459       tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
 460     }
 461 }
 462
 463 static void
 464 do_print  (const void *nodep, VISIT value, int level)
 465 {
 466   if (value == leaf || value == postorder)
 467     {
 468       const char *s = *(const char **) nodep;
 469       size_t len = strlen (s);
 470       size_t cnt;
 471
 472       while (len > 0 && s[len - 1] == '/')
 473         --len;
 474
 475       for (cnt = 0; cnt < len; ++cnt)
 476         if (isalnum (s[cnt]))
 477           break;
 478       if (cnt == len)
 479         return;
 480
 481       if (not_first)
 482         {
 483           putchar (',');
 484           ++column;
 485
 486           if (column > 2 && column + len > 77)
 487             {
 488               fputs ("\n  ", stdout);
 489               column = 2;
 490             }
 491           else
 492             {
 493               putchar (' ');
 494               ++column;
 495             }
 496         }
 497       else
 498         not_first = 1;
 499
 500       fwrite (s, len, 1, stdout);
 501       column += len;
 502     }
 503 }
 504
 505 static void
 506 internal_function
 507 add_known_names (struct gconv_module *node)
 508 {
 509   if (node->left != NULL)
 510     add_known_names (node->left);
 511   if (node->right != NULL)
 512     add_known_names (node->right);
 513   if (node->same != NULL)
 514     add_known_names (node->same);
 515   do
 516     {
 517       if (node->from_pattern == NULL)
 518         {
 519           if (strcmp (node->from_constpfx, "INTERNAL"))
 520             tsearch (node->from_constpfx, &printlist,
 521                      (__compar_fn_t) strverscmp);
 522           if (strcmp (node->to_string, "INTERNAL"))
 523             tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
 524         }
 525       else
 526         if (strcmp (node->from_pattern, "INTERNAL"))
 527           tsearch (node->from_pattern, &printlist, (__compar_fn_t) strverscmp);
 528
 529       node = node->matching;
 530     }
 531   while (node != NULL);
 532 }
 533
 534 static void
 535 internal_function
 536 print_known_names (void)
 537 {
 538   iconv_t h;
 539
 540   /* We must initialize the internal databases first.  */
 541   h = iconv_open ("L1", "L1");
 542   iconv_close (h);
 543
 544   /* First add the aliases.  */
 545   twalk (__gconv_alias_db, insert_print_list);
 546
 547   /* Add the from- and to-names from the known modules.  */
 548   add_known_names (__gconv_modules_db);
 549
 550   fputs (_("\
 551 The following list contain all the coded character sets known.  This does\n\
 552 not necessarily mean that all combinations of these names can be used for\n\
 553 the FROM and TO command line parameters.  One coded character set can be\n\
 554 listed with several different names (aliases).\n\
 555   Some of the names are no plain strings but instead regular expressions and\n\
 556 they match a variety of names which can be given as parameters to the\n\
 557 program.\n\n  "), stdout);
 558
 559   /* Now print the collected names.  */
 560   column = 2;
 561   twalk (printlist, do_print);
 562
 563   if (column != 0)
 564     puts ("");
 565 }