lib/charsets.c

   1 /*
   2    Text conversion from one charset to another.
   3
   4    Copyright (C) 2001-2017
   5    Free Software Foundation, Inc.
   6
   7    Written by:
   8    Walery Studennikov <despair@sama.ru>
   9
  10    This file is part of the Midnight Commander.
  11
  12    The Midnight Commander is free software: you can redistribute it
  13    and/or modify it under the terms of the GNU General Public License as
  14    published by the Free Software Foundation, either version 3 of the License,
  15    or (at your option) any later version.
  16
  17    The Midnight Commander is distributed in the hope that it will be useful,
  18    but WITHOUT ANY WARRANTY; without even the implied warranty of
  19    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20    GNU General Public License for more details.
  21
  22    You should have received a copy of the GNU General Public License
  23    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24  */
  25
  26 /** \file charsets.c
  27  *  \brief Source: Text conversion from one charset to another
  28  */
  29
  30 #include <config.h>
  31
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <string.h>
  35
  36 #include "lib/global.h"
  37 #include "lib/strutil.h"        /* utf-8 functions */
  38 #include "lib/fileloc.h"
  39 #include "lib/util.h"           /* whitespace() */
  40
  41 #include "lib/charsets.h"
  42
  43 /*** global variables ****************************************************************************/
  44
  45 GPtrArray *codepages = NULL;
  46
  47 unsigned char conv_displ[256];
  48 unsigned char conv_input[256];
  49
  50 const char *cp_display = NULL;
  51 const char *cp_source = NULL;
  52
  53 /*** file scope macro definitions ****************************************************************/
  54
  55 #define UNKNCHAR '\001'
  56
  57 #define OTHER_8BIT "Other_8_bit"
  58
  59 /*** file scope type declarations ****************************************************************/
  60
  61 /*** file scope variables ************************************************************************/
  62
  63 /*** file scope functions ************************************************************************/
  64 /* --------------------------------------------------------------------------------------------- */
  65
  66 static codepage_desc *
  67 new_codepage_desc (const char *id, const char *name)
  68 {
  69     codepage_desc *desc;
  70
  71     desc = g_new (codepage_desc, 1);
  72     desc->id = g_strdup (id);
  73     desc->name = g_strdup (name);
  74
  75     return desc;
  76 }
  77
  78 /* --------------------------------------------------------------------------------------------- */
  79
  80 static void
  81 free_codepage_desc (gpointer data, gpointer user_data)
  82 {
  83     codepage_desc *desc = (codepage_desc *) data;
  84     (void) user_data;
  85
  86     g_free (desc->id);
  87     g_free (desc->name);
  88     g_free (desc);
  89 }
  90
  91 /* --------------------------------------------------------------------------------------------- */
  92 /* returns display codepage */
  93
  94 static void
  95 load_codepages_list_from_file (GPtrArray ** list, const char *fname)
  96 {
  97     FILE *f;
  98     char buf[BUF_MEDIUM];
  99     char *default_codepage = NULL;
 100
 101     f = fopen (fname, "r");
 102     if (f == NULL)
 103         return;
 104
 105     while (fgets (buf, sizeof buf, f) != NULL)
 106     {
 107         /* split string into id and cpname */
 108         char *p = buf;
 109         size_t buflen = strlen (buf);
 110
 111         if (*p == '\n' || *p == '\0' || *p == '#')
 112             continue;
 113
 114         if (buflen > 0 && buf[buflen - 1] == '\n')
 115             buf[buflen - 1] = '\0';
 116         while (*p != '\0' && !whitespace (*p))
 117             ++p;
 118         if (*p == '\0')
 119             goto fail;
 120
 121         *p++ = '\0';
 122         g_strstrip (p);
 123         if (*p == '\0')
 124             goto fail;
 125
 126         if (strcmp (buf, "default") == 0)
 127             default_codepage = g_strdup (p);
 128         else
 129         {
 130             const char *id = buf;
 131
 132             if (*list == NULL)
 133             {
 134                 *list = g_ptr_array_sized_new (16);
 135                 g_ptr_array_add (*list, new_codepage_desc (id, p));
 136             }
 137             else
 138             {
 139                 unsigned int i;
 140
 141                 /* whether id is already present in list */
 142                 /* if yes, overwrite description */
 143                 for (i = 0; i < (*list)->len; i++)
 144                 {
 145                     codepage_desc *desc;
 146
 147                     desc = (codepage_desc *) g_ptr_array_index (*list, i);
 148
 149                     if (strcmp (id, desc->id) == 0)
 150                     {
 151                         /* found */
 152                         g_free (desc->name);
 153                         desc->name = g_strdup (p);
 154                         break;
 155                     }
 156                 }
 157
 158                 /* not found */
 159                 if (i == (*list)->len)
 160                     g_ptr_array_add (*list, new_codepage_desc (id, p));
 161             }
 162         }
 163     }
 164
 165     if (default_codepage != NULL)
 166     {
 167         mc_global.display_codepage = get_codepage_index (default_codepage);
 168         g_free (default_codepage);
 169     }
 170
 171   fail:
 172     fclose (f);
 173 }
 174
 175 /* --------------------------------------------------------------------------------------------- */
 176
 177 static char
 178 translate_character (GIConv cd, char c)
 179 {
 180     gchar *tmp_buff = NULL;
 181     gsize bytes_read, bytes_written = 0;
 182     const char *ibuf = &c;
 183     char ch = UNKNCHAR;
 184
 185     int ibuflen = 1;
 186
 187     tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
 188     if (tmp_buff)
 189         ch = tmp_buff[0];
 190     g_free (tmp_buff);
 191     return ch;
 192 }
 193
 194 /* --------------------------------------------------------------------------------------------- */
 195 /*** public functions ****************************************************************************/
 196 /* --------------------------------------------------------------------------------------------- */
 197
 198 void
 199 load_codepages_list (void)
 200 {
 201     char *fname;
 202
 203     /* 1: try load /usr/share/mc/mc.charsets */
 204     fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
 205     load_codepages_list_from_file (&codepages, fname);
 206     g_free (fname);
 207
 208     /* 2: try load /etc/mc/mc.charsets */
 209     fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
 210     load_codepages_list_from_file (&codepages, fname);
 211     g_free (fname);
 212
 213     if (codepages == NULL)
 214     {
 215         /* files are not found, add defaullt codepage */
 216         fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
 217
 218         codepages = g_ptr_array_new ();
 219         g_ptr_array_add (codepages, new_codepage_desc (DEFAULT_CHARSET, _("7-bit ASCII")));
 220     }
 221 }
 222
 223 /* --------------------------------------------------------------------------------------------- */
 224
 225 void
 226 free_codepages_list (void)
 227 {
 228     g_ptr_array_foreach (codepages, free_codepage_desc, NULL);
 229     g_ptr_array_free (codepages, TRUE);
 230     /* NULL-ize pointer to make unit tests happy */
 231     codepages = NULL;
 232 }
 233
 234 /* --------------------------------------------------------------------------------------------- */
 235
 236 const char *
 237 get_codepage_id (const int n)
 238 {
 239     return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
 240 }
 241
 242 /* --------------------------------------------------------------------------------------------- */
 243
 244 int
 245 get_codepage_index (const char *id)
 246 {
 247     size_t i;
 248     if (strcmp (id, OTHER_8BIT) == 0)
 249         return -1;
 250     if (codepages == NULL)
 251         return -1;
 252     for (i = 0; i < codepages->len; i++)
 253         if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
 254             return i;
 255     return -1;
 256 }
 257
 258 /* --------------------------------------------------------------------------------------------- */
 259 /** Check if specified encoding can be used in mc.
 260  * @param encoding name of encoding
 261  * @return TRUE if encoding is supported by mc, FALSE otherwise
 262  */
 263
 264 gboolean
 265 is_supported_encoding (const char *encoding)
 266 {
 267     gboolean result = FALSE;
 268     guint t;
 269
 270     for (t = 0; t < codepages->len; t++)
 271     {
 272         const char *id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
 273         result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
 274     }
 275
 276     return result;
 277 }
 278
 279 /* --------------------------------------------------------------------------------------------- */
 280
 281 char *
 282 init_translation_table (int cpsource, int cpdisplay)
 283 {
 284     int i;
 285     GIConv cd;
 286
 287     /* Fill inpit <-> display tables */
 288
 289     if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
 290     {
 291         for (i = 0; i <= 255; ++i)
 292         {
 293             conv_displ[i] = i;
 294             conv_input[i] = i;
 295             cp_source = cp_display;
 296         }
 297         return NULL;
 298     }
 299
 300     for (i = 0; i <= 127; ++i)
 301     {
 302         conv_displ[i] = i;
 303         conv_input[i] = i;
 304     }
 305     cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
 306     cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
 307
 308     /* display <- inpit table */
 309
 310     cd = g_iconv_open (cp_display, cp_source);
 311     if (cd == INVALID_CONV)
 312         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
 313
 314     for (i = 128; i <= 255; ++i)
 315         conv_displ[i] = translate_character (cd, i);
 316
 317     g_iconv_close (cd);
 318
 319     /* inpit <- display table */
 320
 321     cd = g_iconv_open (cp_source, cp_display);
 322     if (cd == INVALID_CONV)
 323         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
 324
 325     for (i = 128; i <= 255; ++i)
 326     {
 327         unsigned char ch;
 328         ch = translate_character (cd, i);
 329         conv_input[i] = (ch == UNKNCHAR) ? i : ch;
 330     }
 331
 332     g_iconv_close (cd);
 333
 334     return NULL;
 335 }
 336
 337 /* --------------------------------------------------------------------------------------------- */
 338
 339 void
 340 convert_to_display (char *str)
 341 {
 342     if (!str)
 343         return;
 344
 345     while (*str)
 346     {
 347         *str = conv_displ[(unsigned char) *str];
 348         str++;
 349     }
 350 }
 351
 352 /* --------------------------------------------------------------------------------------------- */
 353
 354 GString *
 355 str_convert_to_display (const char *str)
 356 {
 357     return str_nconvert_to_display (str, -1);
 358
 359 }
 360
 361 /* --------------------------------------------------------------------------------------------- */
 362
 363 GString *
 364 str_nconvert_to_display (const char *str, int len)
 365 {
 366     GString *buff;
 367     GIConv conv;
 368
 369     if (!str)
 370         return g_string_new ("");
 371
 372     if (cp_display == cp_source)
 373         return g_string_new (str);
 374
 375     conv = str_crt_conv_from (cp_source);
 376
 377     buff = g_string_new ("");
 378     str_nconvert (conv, str, len, buff);
 379     str_close_conv (conv);
 380     return buff;
 381 }
 382
 383 /* --------------------------------------------------------------------------------------------- */
 384
 385 void
 386 convert_from_input (char *str)
 387 {
 388     if (!str)
 389         return;
 390
 391     while (*str)
 392     {
 393         *str = conv_input[(unsigned char) *str];
 394         str++;
 395     }
 396 }
 397
 398 /* --------------------------------------------------------------------------------------------- */
 399
 400 GString *
 401 str_convert_to_input (const char *str)
 402 {
 403     return str_nconvert_to_input (str, -1);
 404 }
 405
 406 /* --------------------------------------------------------------------------------------------- */
 407
 408 GString *
 409 str_nconvert_to_input (const char *str, int len)
 410 {
 411     GString *buff;
 412     GIConv conv;
 413
 414     if (!str)
 415         return g_string_new ("");
 416
 417     if (cp_display == cp_source)
 418         return g_string_new (str);
 419
 420     conv = str_crt_conv_to (cp_source);
 421
 422     buff = g_string_new ("");
 423     str_nconvert (conv, str, len, buff);
 424     str_close_conv (conv);
 425     return buff;
 426 }
 427
 428 /* --------------------------------------------------------------------------------------------- */
 429
 430 unsigned char
 431 convert_from_utf_to_current (const char *str)
 432 {
 433     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 434     unsigned char ch = '.';
 435     GIConv conv;
 436     const char *cp_to;
 437
 438     if (str == NULL)
 439         return '.';
 440
 441     cp_to = get_codepage_id (mc_global.source_codepage);
 442     conv = str_crt_conv_to (cp_to);
 443
 444     if (conv != INVALID_CONV)
 445     {
 446         switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
 447         {
 448         case ESTR_SUCCESS:
 449             ch = buf_ch[0];
 450             break;
 451         case ESTR_PROBLEM:
 452         case ESTR_FAILURE:
 453             ch = '.';
 454             break;
 455         default:
 456             break;
 457         }
 458         str_close_conv (conv);
 459     }
 460
 461     return ch;
 462 }
 463
 464 /* --------------------------------------------------------------------------------------------- */
 465
 466 unsigned char
 467 convert_from_utf_to_current_c (int input_char, GIConv conv)
 468 {
 469     unsigned char str[UTF8_CHAR_LEN + 1];
 470     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 471     unsigned char ch = '.';
 472     int res;
 473
 474     res = g_unichar_to_utf8 (input_char, (char *) str);
 475     if (res == 0)
 476         return ch;
 477
 478     str[res] = '\0';
 479
 480     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 481     {
 482     case ESTR_SUCCESS:
 483         ch = buf_ch[0];
 484         break;
 485     case ESTR_PROBLEM:
 486     case ESTR_FAILURE:
 487         ch = '.';
 488         break;
 489     default:
 490         break;
 491     }
 492
 493     return ch;
 494 }
 495
 496 /* --------------------------------------------------------------------------------------------- */
 497
 498 int
 499 convert_from_8bit_to_utf_c (char input_char, GIConv conv)
 500 {
 501     unsigned char str[2];
 502     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 503     int ch;
 504
 505     str[0] = (unsigned char) input_char;
 506     str[1] = '\0';
 507
 508     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 509     {
 510     case ESTR_SUCCESS:
 511         {
 512             int res;
 513
 514             res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 515             ch = res >= 0 ? res : buf_ch[0];
 516             break;
 517         }
 518     case ESTR_PROBLEM:
 519     case ESTR_FAILURE:
 520     default:
 521         ch = '.';
 522         break;
 523     }
 524
 525     return ch;
 526 }
 527
 528 /* --------------------------------------------------------------------------------------------- */
 529
 530 int
 531 convert_from_8bit_to_utf_c2 (char input_char)
 532 {
 533     unsigned char str[2];
 534     int ch = '.';
 535     GIConv conv;
 536     const char *cp_from;
 537
 538     str[0] = (unsigned char) input_char;
 539     str[1] = '\0';
 540
 541     cp_from = get_codepage_id (mc_global.source_codepage);
 542     conv = str_crt_conv_to (cp_from);
 543
 544     if (conv != INVALID_CONV)
 545     {
 546         unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 547
 548         switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 549         {
 550         case ESTR_SUCCESS:
 551             {
 552                 int res;
 553
 554                 res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 555                 ch = res >= 0 ? res : buf_ch[0];
 556                 break;
 557             }
 558         case ESTR_PROBLEM:
 559         case ESTR_FAILURE:
 560         default:
 561             ch = '.';
 562             break;
 563         }
 564         str_close_conv (conv);
 565     }
 566
 567     return ch;
 568 }
 569
 570 /* --------------------------------------------------------------------------------------------- */