lib/charsets.c

   1 /*
   2    Text conversion from one charset to another.
   3
   4    Copyright (C) 2001-2015
   5    Free Software Foundation, Inc.
   6
   7    Written by:
   8    Walery Studennikov <despair@sama.ru>
   9
  10    This file is part of the Midnight Commander.
  11
  12    The Midnight Commander is free software: you can redistribute it
  13    and/or modify it under the terms of the GNU General Public License as
  14    published by the Free Software Foundation, either version 3 of the License,
  15    or (at your option) any later version.
  16
  17    The Midnight Commander is distributed in the hope that it will be useful,
  18    but WITHOUT ANY WARRANTY; without even the implied warranty of
  19    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20    GNU General Public License for more details.
  21
  22    You should have received a copy of the GNU General Public License
  23    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24  */
  25
  26 /** \file charsets.c
  27  *  \brief Source: Text conversion from one charset to another
  28  */
  29
  30 #include <config.h>
  31
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <string.h>
  35
  36 #include "lib/global.h"
  37 #include "lib/strutil.h"        /* utf-8 functions */
  38 #include "lib/fileloc.h"
  39 #include "lib/charsets.h"
  40
  41 /*** global variables ****************************************************************************/
  42
  43 GPtrArray *codepages = NULL;
  44
  45 unsigned char conv_displ[256];
  46 unsigned char conv_input[256];
  47
  48 const char *cp_display = NULL;
  49 const char *cp_source = NULL;
  50
  51 /*** file scope macro definitions ****************************************************************/
  52
  53 #define UNKNCHAR '\001'
  54
  55 #define OTHER_8BIT "Other_8_bit"
  56
  57 /*
  58  * FIXME: This assumes that ASCII is always the first encoding
  59  * in mc.charsets
  60  */
  61 #define CP_ASCII 0
  62
  63 /*** file scope type declarations ****************************************************************/
  64
  65 /*** file scope variables ************************************************************************/
  66
  67 /*** file scope functions ************************************************************************/
  68 /* --------------------------------------------------------------------------------------------- */
  69
  70 static codepage_desc *
  71 new_codepage_desc (const char *id, const char *name)
  72 {
  73     codepage_desc *desc;
  74
  75     desc = g_new (codepage_desc, 1);
  76     desc->id = g_strdup (id);
  77     desc->name = g_strdup (name);
  78
  79     return desc;
  80 }
  81
  82 /* --------------------------------------------------------------------------------------------- */
  83
  84 static void
  85 free_codepage_desc (gpointer data, gpointer user_data)
  86 {
  87     codepage_desc *desc = (codepage_desc *) data;
  88     (void) user_data;
  89
  90     g_free (desc->id);
  91     g_free (desc->name);
  92     g_free (desc);
  93 }
  94
  95 /* --------------------------------------------------------------------------------------------- */
  96 /* returns display codepage */
  97
  98 static void
  99 load_codepages_list_from_file (GPtrArray ** list, const char *fname)
 100 {
 101     FILE *f;
 102     char buf[BUF_MEDIUM];
 103     char *default_codepage = NULL;
 104
 105     f = fopen (fname, "r");
 106     if (f == NULL)
 107         return;
 108
 109     while (fgets (buf, sizeof buf, f) != NULL)
 110     {
 111         /* split string into id and cpname */
 112         char *p = buf;
 113         size_t buflen = strlen (buf);
 114
 115         if (*p == '\n' || *p == '\0' || *p == '#')
 116             continue;
 117
 118         if (buflen > 0 && buf[buflen - 1] == '\n')
 119             buf[buflen - 1] = '\0';
 120         while (*p != '\t' && *p != ' ' && *p != '\0')
 121             ++p;
 122         if (*p == '\0')
 123             goto fail;
 124
 125         *p++ = '\0';
 126         g_strstrip (p);
 127         if (*p == '\0')
 128             goto fail;
 129
 130         if (strcmp (buf, "default") == 0)
 131             default_codepage = g_strdup (p);
 132         else
 133         {
 134             const char *id = buf;
 135
 136             if (*list == NULL)
 137             {
 138                 *list = g_ptr_array_sized_new (16);
 139                 g_ptr_array_add (*list, new_codepage_desc (id, p));
 140             }
 141             else
 142             {
 143                 unsigned int i;
 144
 145                 /* whether id is already present in list */
 146                 /* if yes, overwrite description */
 147                 for (i = 0; i < (*list)->len; i++)
 148                 {
 149                     codepage_desc *desc;
 150
 151                     desc = (codepage_desc *) g_ptr_array_index (*list, i);
 152
 153                     if (strcmp (id, desc->id) == 0)
 154                     {
 155                         /* found */
 156                         g_free (desc->name);
 157                         desc->name = g_strdup (p);
 158                         break;
 159                     }
 160                 }
 161
 162                 /* not found */
 163                 if (i == (*list)->len)
 164                     g_ptr_array_add (*list, new_codepage_desc (id, p));
 165             }
 166         }
 167     }
 168
 169     if (default_codepage != NULL)
 170     {
 171         mc_global.display_codepage = get_codepage_index (default_codepage);
 172         g_free (default_codepage);
 173     }
 174
 175   fail:
 176     fclose (f);
 177 }
 178
 179 /* --------------------------------------------------------------------------------------------- */
 180
 181 static char
 182 translate_character (GIConv cd, char c)
 183 {
 184     gchar *tmp_buff = NULL;
 185     gsize bytes_read, bytes_written = 0;
 186     const char *ibuf = &c;
 187     char ch = UNKNCHAR;
 188
 189     int ibuflen = 1;
 190
 191     tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
 192     if (tmp_buff)
 193         ch = tmp_buff[0];
 194     g_free (tmp_buff);
 195     return ch;
 196 }
 197
 198 /* --------------------------------------------------------------------------------------------- */
 199 /*** public functions ****************************************************************************/
 200 /* --------------------------------------------------------------------------------------------- */
 201
 202 void
 203 load_codepages_list (void)
 204 {
 205     char *fname;
 206
 207     /* 1: try load /usr/share/mc/mc.charsets */
 208     fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
 209     load_codepages_list_from_file (&codepages, fname);
 210     g_free (fname);
 211
 212     /* 2: try load /etc/mc/mc.charsets */
 213     fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
 214     load_codepages_list_from_file (&codepages, fname);
 215     g_free (fname);
 216
 217     if (codepages == NULL)
 218     {
 219         /* files are not found, add defaullt codepage */
 220         fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
 221
 222         codepages = g_ptr_array_new ();
 223         g_ptr_array_add (codepages, new_codepage_desc (DEFAULT_CHARSET, _("7-bit ASCII")));
 224     }
 225 }
 226
 227 /* --------------------------------------------------------------------------------------------- */
 228
 229 void
 230 free_codepages_list (void)
 231 {
 232     g_ptr_array_foreach (codepages, free_codepage_desc, NULL);
 233     g_ptr_array_free (codepages, TRUE);
 234 }
 235
 236 /* --------------------------------------------------------------------------------------------- */
 237
 238 const char *
 239 get_codepage_id (const int n)
 240 {
 241     return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
 242 }
 243
 244 /* --------------------------------------------------------------------------------------------- */
 245
 246 int
 247 get_codepage_index (const char *id)
 248 {
 249     size_t i;
 250     if (strcmp (id, OTHER_8BIT) == 0)
 251         return -1;
 252     if (codepages == NULL)
 253         return -1;
 254     for (i = 0; i < codepages->len; i++)
 255         if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
 256             return i;
 257     return -1;
 258 }
 259
 260 /* --------------------------------------------------------------------------------------------- */
 261 /** Check if specified encoding can be used in mc.
 262  * @param encoding name of encoding
 263  * @return TRUE if encoding is supported by mc, FALSE otherwise
 264  */
 265
 266 gboolean
 267 is_supported_encoding (const char *encoding)
 268 {
 269     gboolean result = FALSE;
 270     guint t;
 271
 272     for (t = 0; t < codepages->len; t++)
 273     {
 274         const char *id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
 275         result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
 276     }
 277
 278     return result;
 279 }
 280
 281 /* --------------------------------------------------------------------------------------------- */
 282
 283 char *
 284 init_translation_table (int cpsource, int cpdisplay)
 285 {
 286     int i;
 287     GIConv cd;
 288
 289     /* Fill inpit <-> display tables */
 290
 291     if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
 292     {
 293         for (i = 0; i <= 255; ++i)
 294         {
 295             conv_displ[i] = i;
 296             conv_input[i] = i;
 297             cp_source = cp_display;
 298         }
 299         return NULL;
 300     }
 301
 302     for (i = 0; i <= 127; ++i)
 303     {
 304         conv_displ[i] = i;
 305         conv_input[i] = i;
 306     }
 307     cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
 308     cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
 309
 310     /* display <- inpit table */
 311
 312     cd = g_iconv_open (cp_display, cp_source);
 313     if (cd == INVALID_CONV)
 314         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
 315
 316     for (i = 128; i <= 255; ++i)
 317         conv_displ[i] = translate_character (cd, i);
 318
 319     g_iconv_close (cd);
 320
 321     /* inpit <- display table */
 322
 323     cd = g_iconv_open (cp_source, cp_display);
 324     if (cd == INVALID_CONV)
 325         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
 326
 327     for (i = 128; i <= 255; ++i)
 328     {
 329         unsigned char ch;
 330         ch = translate_character (cd, i);
 331         conv_input[i] = (ch == UNKNCHAR) ? i : ch;
 332     }
 333
 334     g_iconv_close (cd);
 335
 336     return NULL;
 337 }
 338
 339 /* --------------------------------------------------------------------------------------------- */
 340
 341 void
 342 convert_to_display (char *str)
 343 {
 344     if (!str)
 345         return;
 346
 347     while (*str)
 348     {
 349         *str = conv_displ[(unsigned char) *str];
 350         str++;
 351     }
 352 }
 353
 354 /* --------------------------------------------------------------------------------------------- */
 355
 356 GString *
 357 str_convert_to_display (char *str)
 358 {
 359     return str_nconvert_to_display (str, -1);
 360
 361 }
 362
 363 /* --------------------------------------------------------------------------------------------- */
 364
 365 GString *
 366 str_nconvert_to_display (char *str, int len)
 367 {
 368     GString *buff;
 369     GIConv conv;
 370
 371     if (!str)
 372         return g_string_new ("");
 373
 374     if (cp_display == cp_source)
 375         return g_string_new (str);
 376
 377     conv = str_crt_conv_from (cp_source);
 378
 379     buff = g_string_new ("");
 380     str_nconvert (conv, str, len, buff);
 381     str_close_conv (conv);
 382     return buff;
 383 }
 384
 385 /* --------------------------------------------------------------------------------------------- */
 386
 387 void
 388 convert_from_input (char *str)
 389 {
 390     if (!str)
 391         return;
 392
 393     while (*str)
 394     {
 395         *str = conv_input[(unsigned char) *str];
 396         str++;
 397     }
 398 }
 399
 400 /* --------------------------------------------------------------------------------------------- */
 401
 402 GString *
 403 str_convert_to_input (char *str)
 404 {
 405     return str_nconvert_to_input (str, -1);
 406 }
 407
 408 /* --------------------------------------------------------------------------------------------- */
 409
 410 GString *
 411 str_nconvert_to_input (char *str, int len)
 412 {
 413     GString *buff;
 414     GIConv conv;
 415
 416     if (!str)
 417         return g_string_new ("");
 418
 419     if (cp_display == cp_source)
 420         return g_string_new (str);
 421
 422     conv = str_crt_conv_to (cp_source);
 423
 424     buff = g_string_new ("");
 425     str_nconvert (conv, str, len, buff);
 426     str_close_conv (conv);
 427     return buff;
 428 }
 429
 430 /* --------------------------------------------------------------------------------------------- */
 431
 432 unsigned char
 433 convert_from_utf_to_current (const char *str)
 434 {
 435     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 436     unsigned char ch = '.';
 437     GIConv conv;
 438     const char *cp_to;
 439
 440     if (str == NULL)
 441         return '.';
 442
 443     cp_to = get_codepage_id (mc_global.source_codepage);
 444     conv = str_crt_conv_to (cp_to);
 445
 446     if (conv != INVALID_CONV)
 447     {
 448         switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
 449         {
 450         case ESTR_SUCCESS:
 451             ch = buf_ch[0];
 452             break;
 453         case ESTR_PROBLEM:
 454         case ESTR_FAILURE:
 455             ch = '.';
 456             break;
 457         default:
 458             break;
 459         }
 460         str_close_conv (conv);
 461     }
 462
 463     return ch;
 464 }
 465
 466 /* --------------------------------------------------------------------------------------------- */
 467
 468 unsigned char
 469 convert_from_utf_to_current_c (const int input_char, GIConv conv)
 470 {
 471     unsigned char str[UTF8_CHAR_LEN + 1];
 472     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 473     unsigned char ch = '.';
 474     int res;
 475
 476     res = g_unichar_to_utf8 (input_char, (char *) str);
 477     if (res == 0)
 478         return ch;
 479
 480     str[res] = '\0';
 481
 482     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 483     {
 484     case ESTR_SUCCESS:
 485         ch = buf_ch[0];
 486         break;
 487     case ESTR_PROBLEM:
 488     case ESTR_FAILURE:
 489         ch = '.';
 490         break;
 491     default:
 492         break;
 493     }
 494
 495     return ch;
 496 }
 497
 498 /* --------------------------------------------------------------------------------------------- */
 499
 500 int
 501 convert_from_8bit_to_utf_c (const char input_char, GIConv conv)
 502 {
 503     unsigned char str[2];
 504     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 505     int ch;
 506
 507     str[0] = (unsigned char) input_char;
 508     str[1] = '\0';
 509
 510     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 511     {
 512     case ESTR_SUCCESS:
 513         {
 514             int res;
 515
 516             res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 517             ch = res >= 0 ? res : buf_ch[0];
 518             break;
 519         }
 520     case ESTR_PROBLEM:
 521     case ESTR_FAILURE:
 522     default:
 523         ch = '.';
 524         break;
 525     }
 526
 527     return ch;
 528 }
 529
 530 /* --------------------------------------------------------------------------------------------- */
 531
 532 int
 533 convert_from_8bit_to_utf_c2 (const char input_char)
 534 {
 535     unsigned char str[2];
 536     int ch = '.';
 537     GIConv conv;
 538     const char *cp_from;
 539
 540     str[0] = (unsigned char) input_char;
 541     str[1] = '\0';
 542
 543     cp_from = get_codepage_id (mc_global.source_codepage);
 544     conv = str_crt_conv_to (cp_from);
 545
 546     if (conv != INVALID_CONV)
 547     {
 548         unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 549
 550         switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 551         {
 552         case ESTR_SUCCESS:
 553             {
 554                 int res;
 555
 556                 res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 557                 ch = res >= 0 ? res : buf_ch[0];
 558                 break;
 559             }
 560         case ESTR_PROBLEM:
 561         case ESTR_FAILURE:
 562         default:
 563             ch = '.';
 564             break;
 565         }
 566         str_close_conv (conv);
 567     }
 568
 569     return ch;
 570 }
 571
 572 /* --------------------------------------------------------------------------------------------- */