lib/charsets.c

   1 /*
   2    Text conversion from one charset to another.
   3
   4    Copyright (C) 2001-2016
   5    Free Software Foundation, Inc.
   6
   7    Written by:
   8    Walery Studennikov <despair@sama.ru>
   9
  10    This file is part of the Midnight Commander.
  11
  12    The Midnight Commander is free software: you can redistribute it
  13    and/or modify it under the terms of the GNU General Public License as
  14    published by the Free Software Foundation, either version 3 of the License,
  15    or (at your option) any later version.
  16
  17    The Midnight Commander is distributed in the hope that it will be useful,
  18    but WITHOUT ANY WARRANTY; without even the implied warranty of
  19    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20    GNU General Public License for more details.
  21
  22    You should have received a copy of the GNU General Public License
  23    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24  */
  25
  26 /** \file charsets.c
  27  *  \brief Source: Text conversion from one charset to another
  28  */
  29
  30 #include <config.h>
  31
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <string.h>
  35
  36 #include "lib/global.h"
  37 #include "lib/strutil.h"        /* utf-8 functions */
  38 #include "lib/fileloc.h"
  39 #include "lib/charsets.h"
  40
  41 /*** global variables ****************************************************************************/
  42
  43 GPtrArray *codepages = NULL;
  44
  45 unsigned char conv_displ[256];
  46 unsigned char conv_input[256];
  47
  48 const char *cp_display = NULL;
  49 const char *cp_source = NULL;
  50
  51 /*** file scope macro definitions ****************************************************************/
  52
  53 #define UNKNCHAR '\001'
  54
  55 #define OTHER_8BIT "Other_8_bit"
  56
  57 /*** file scope type declarations ****************************************************************/
  58
  59 /*** file scope variables ************************************************************************/
  60
  61 /*** file scope functions ************************************************************************/
  62 /* --------------------------------------------------------------------------------------------- */
  63
  64 static codepage_desc *
  65 new_codepage_desc (const char *id, const char *name)
  66 {
  67     codepage_desc *desc;
  68
  69     desc = g_new (codepage_desc, 1);
  70     desc->id = g_strdup (id);
  71     desc->name = g_strdup (name);
  72
  73     return desc;
  74 }
  75
  76 /* --------------------------------------------------------------------------------------------- */
  77
  78 static void
  79 free_codepage_desc (gpointer data, gpointer user_data)
  80 {
  81     codepage_desc *desc = (codepage_desc *) data;
  82     (void) user_data;
  83
  84     g_free (desc->id);
  85     g_free (desc->name);
  86     g_free (desc);
  87 }
  88
  89 /* --------------------------------------------------------------------------------------------- */
  90 /* returns display codepage */
  91
  92 static void
  93 load_codepages_list_from_file (GPtrArray ** list, const char *fname)
  94 {
  95     FILE *f;
  96     char buf[BUF_MEDIUM];
  97     char *default_codepage = NULL;
  98
  99     f = fopen (fname, "r");
 100     if (f == NULL)
 101         return;
 102
 103     while (fgets (buf, sizeof buf, f) != NULL)
 104     {
 105         /* split string into id and cpname */
 106         char *p = buf;
 107         size_t buflen = strlen (buf);
 108
 109         if (*p == '\n' || *p == '\0' || *p == '#')
 110             continue;
 111
 112         if (buflen > 0 && buf[buflen - 1] == '\n')
 113             buf[buflen - 1] = '\0';
 114         while (*p != '\t' && *p != ' ' && *p != '\0')
 115             ++p;
 116         if (*p == '\0')
 117             goto fail;
 118
 119         *p++ = '\0';
 120         g_strstrip (p);
 121         if (*p == '\0')
 122             goto fail;
 123
 124         if (strcmp (buf, "default") == 0)
 125             default_codepage = g_strdup (p);
 126         else
 127         {
 128             const char *id = buf;
 129
 130             if (*list == NULL)
 131             {
 132                 *list = g_ptr_array_sized_new (16);
 133                 g_ptr_array_add (*list, new_codepage_desc (id, p));
 134             }
 135             else
 136             {
 137                 unsigned int i;
 138
 139                 /* whether id is already present in list */
 140                 /* if yes, overwrite description */
 141                 for (i = 0; i < (*list)->len; i++)
 142                 {
 143                     codepage_desc *desc;
 144
 145                     desc = (codepage_desc *) g_ptr_array_index (*list, i);
 146
 147                     if (strcmp (id, desc->id) == 0)
 148                     {
 149                         /* found */
 150                         g_free (desc->name);
 151                         desc->name = g_strdup (p);
 152                         break;
 153                     }
 154                 }
 155
 156                 /* not found */
 157                 if (i == (*list)->len)
 158                     g_ptr_array_add (*list, new_codepage_desc (id, p));
 159             }
 160         }
 161     }
 162
 163     if (default_codepage != NULL)
 164     {
 165         mc_global.display_codepage = get_codepage_index (default_codepage);
 166         g_free (default_codepage);
 167     }
 168
 169   fail:
 170     fclose (f);
 171 }
 172
 173 /* --------------------------------------------------------------------------------------------- */
 174
 175 static char
 176 translate_character (GIConv cd, char c)
 177 {
 178     gchar *tmp_buff = NULL;
 179     gsize bytes_read, bytes_written = 0;
 180     const char *ibuf = &c;
 181     char ch = UNKNCHAR;
 182
 183     int ibuflen = 1;
 184
 185     tmp_buff = g_convert_with_iconv (ibuf, ibuflen, cd, &bytes_read, &bytes_written, NULL);
 186     if (tmp_buff)
 187         ch = tmp_buff[0];
 188     g_free (tmp_buff);
 189     return ch;
 190 }
 191
 192 /* --------------------------------------------------------------------------------------------- */
 193 /*** public functions ****************************************************************************/
 194 /* --------------------------------------------------------------------------------------------- */
 195
 196 void
 197 load_codepages_list (void)
 198 {
 199     char *fname;
 200
 201     /* 1: try load /usr/share/mc/mc.charsets */
 202     fname = g_build_filename (mc_global.share_data_dir, CHARSETS_LIST, (char *) NULL);
 203     load_codepages_list_from_file (&codepages, fname);
 204     g_free (fname);
 205
 206     /* 2: try load /etc/mc/mc.charsets */
 207     fname = g_build_filename (mc_global.sysconfig_dir, CHARSETS_LIST, (char *) NULL);
 208     load_codepages_list_from_file (&codepages, fname);
 209     g_free (fname);
 210
 211     if (codepages == NULL)
 212     {
 213         /* files are not found, add defaullt codepage */
 214         fprintf (stderr, "%s\n", _("Warning: cannot load codepages list"));
 215
 216         codepages = g_ptr_array_new ();
 217         g_ptr_array_add (codepages, new_codepage_desc (DEFAULT_CHARSET, _("7-bit ASCII")));
 218     }
 219 }
 220
 221 /* --------------------------------------------------------------------------------------------- */
 222
 223 void
 224 free_codepages_list (void)
 225 {
 226     g_ptr_array_foreach (codepages, free_codepage_desc, NULL);
 227     g_ptr_array_free (codepages, TRUE);
 228     /* NULL-ize pointer to make unit tests happy */
 229     codepages = NULL;
 230 }
 231
 232 /* --------------------------------------------------------------------------------------------- */
 233
 234 const char *
 235 get_codepage_id (const int n)
 236 {
 237     return (n < 0) ? OTHER_8BIT : ((codepage_desc *) g_ptr_array_index (codepages, n))->id;
 238 }
 239
 240 /* --------------------------------------------------------------------------------------------- */
 241
 242 int
 243 get_codepage_index (const char *id)
 244 {
 245     size_t i;
 246     if (strcmp (id, OTHER_8BIT) == 0)
 247         return -1;
 248     if (codepages == NULL)
 249         return -1;
 250     for (i = 0; i < codepages->len; i++)
 251         if (strcmp (id, ((codepage_desc *) g_ptr_array_index (codepages, i))->id) == 0)
 252             return i;
 253     return -1;
 254 }
 255
 256 /* --------------------------------------------------------------------------------------------- */
 257 /** Check if specified encoding can be used in mc.
 258  * @param encoding name of encoding
 259  * @return TRUE if encoding is supported by mc, FALSE otherwise
 260  */
 261
 262 gboolean
 263 is_supported_encoding (const char *encoding)
 264 {
 265     gboolean result = FALSE;
 266     guint t;
 267
 268     for (t = 0; t < codepages->len; t++)
 269     {
 270         const char *id = ((codepage_desc *) g_ptr_array_index (codepages, t))->id;
 271         result |= (g_ascii_strncasecmp (encoding, id, strlen (id)) == 0);
 272     }
 273
 274     return result;
 275 }
 276
 277 /* --------------------------------------------------------------------------------------------- */
 278
 279 char *
 280 init_translation_table (int cpsource, int cpdisplay)
 281 {
 282     int i;
 283     GIConv cd;
 284
 285     /* Fill inpit <-> display tables */
 286
 287     if (cpsource < 0 || cpdisplay < 0 || cpsource == cpdisplay)
 288     {
 289         for (i = 0; i <= 255; ++i)
 290         {
 291             conv_displ[i] = i;
 292             conv_input[i] = i;
 293             cp_source = cp_display;
 294         }
 295         return NULL;
 296     }
 297
 298     for (i = 0; i <= 127; ++i)
 299     {
 300         conv_displ[i] = i;
 301         conv_input[i] = i;
 302     }
 303     cp_source = ((codepage_desc *) g_ptr_array_index (codepages, cpsource))->id;
 304     cp_display = ((codepage_desc *) g_ptr_array_index (codepages, cpdisplay))->id;
 305
 306     /* display <- inpit table */
 307
 308     cd = g_iconv_open (cp_display, cp_source);
 309     if (cd == INVALID_CONV)
 310         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_source, cp_display);
 311
 312     for (i = 128; i <= 255; ++i)
 313         conv_displ[i] = translate_character (cd, i);
 314
 315     g_iconv_close (cd);
 316
 317     /* inpit <- display table */
 318
 319     cd = g_iconv_open (cp_source, cp_display);
 320     if (cd == INVALID_CONV)
 321         return g_strdup_printf (_("Cannot translate from %s to %s"), cp_display, cp_source);
 322
 323     for (i = 128; i <= 255; ++i)
 324     {
 325         unsigned char ch;
 326         ch = translate_character (cd, i);
 327         conv_input[i] = (ch == UNKNCHAR) ? i : ch;
 328     }
 329
 330     g_iconv_close (cd);
 331
 332     return NULL;
 333 }
 334
 335 /* --------------------------------------------------------------------------------------------- */
 336
 337 void
 338 convert_to_display (char *str)
 339 {
 340     if (!str)
 341         return;
 342
 343     while (*str)
 344     {
 345         *str = conv_displ[(unsigned char) *str];
 346         str++;
 347     }
 348 }
 349
 350 /* --------------------------------------------------------------------------------------------- */
 351
 352 GString *
 353 str_convert_to_display (const char *str)
 354 {
 355     return str_nconvert_to_display (str, -1);
 356
 357 }
 358
 359 /* --------------------------------------------------------------------------------------------- */
 360
 361 GString *
 362 str_nconvert_to_display (const char *str, int len)
 363 {
 364     GString *buff;
 365     GIConv conv;
 366
 367     if (!str)
 368         return g_string_new ("");
 369
 370     if (cp_display == cp_source)
 371         return g_string_new (str);
 372
 373     conv = str_crt_conv_from (cp_source);
 374
 375     buff = g_string_new ("");
 376     str_nconvert (conv, str, len, buff);
 377     str_close_conv (conv);
 378     return buff;
 379 }
 380
 381 /* --------------------------------------------------------------------------------------------- */
 382
 383 void
 384 convert_from_input (char *str)
 385 {
 386     if (!str)
 387         return;
 388
 389     while (*str)
 390     {
 391         *str = conv_input[(unsigned char) *str];
 392         str++;
 393     }
 394 }
 395
 396 /* --------------------------------------------------------------------------------------------- */
 397
 398 GString *
 399 str_convert_to_input (const char *str)
 400 {
 401     return str_nconvert_to_input (str, -1);
 402 }
 403
 404 /* --------------------------------------------------------------------------------------------- */
 405
 406 GString *
 407 str_nconvert_to_input (const char *str, int len)
 408 {
 409     GString *buff;
 410     GIConv conv;
 411
 412     if (!str)
 413         return g_string_new ("");
 414
 415     if (cp_display == cp_source)
 416         return g_string_new (str);
 417
 418     conv = str_crt_conv_to (cp_source);
 419
 420     buff = g_string_new ("");
 421     str_nconvert (conv, str, len, buff);
 422     str_close_conv (conv);
 423     return buff;
 424 }
 425
 426 /* --------------------------------------------------------------------------------------------- */
 427
 428 unsigned char
 429 convert_from_utf_to_current (const char *str)
 430 {
 431     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 432     unsigned char ch = '.';
 433     GIConv conv;
 434     const char *cp_to;
 435
 436     if (str == NULL)
 437         return '.';
 438
 439     cp_to = get_codepage_id (mc_global.source_codepage);
 440     conv = str_crt_conv_to (cp_to);
 441
 442     if (conv != INVALID_CONV)
 443     {
 444         switch (str_translate_char (conv, str, -1, (char *) buf_ch, sizeof (buf_ch)))
 445         {
 446         case ESTR_SUCCESS:
 447             ch = buf_ch[0];
 448             break;
 449         case ESTR_PROBLEM:
 450         case ESTR_FAILURE:
 451             ch = '.';
 452             break;
 453         default:
 454             break;
 455         }
 456         str_close_conv (conv);
 457     }
 458
 459     return ch;
 460 }
 461
 462 /* --------------------------------------------------------------------------------------------- */
 463
 464 unsigned char
 465 convert_from_utf_to_current_c (int input_char, GIConv conv)
 466 {
 467     unsigned char str[UTF8_CHAR_LEN + 1];
 468     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 469     unsigned char ch = '.';
 470     int res;
 471
 472     res = g_unichar_to_utf8 (input_char, (char *) str);
 473     if (res == 0)
 474         return ch;
 475
 476     str[res] = '\0';
 477
 478     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 479     {
 480     case ESTR_SUCCESS:
 481         ch = buf_ch[0];
 482         break;
 483     case ESTR_PROBLEM:
 484     case ESTR_FAILURE:
 485         ch = '.';
 486         break;
 487     default:
 488         break;
 489     }
 490
 491     return ch;
 492 }
 493
 494 /* --------------------------------------------------------------------------------------------- */
 495
 496 int
 497 convert_from_8bit_to_utf_c (char input_char, GIConv conv)
 498 {
 499     unsigned char str[2];
 500     unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 501     int ch;
 502
 503     str[0] = (unsigned char) input_char;
 504     str[1] = '\0';
 505
 506     switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 507     {
 508     case ESTR_SUCCESS:
 509         {
 510             int res;
 511
 512             res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 513             ch = res >= 0 ? res : buf_ch[0];
 514             break;
 515         }
 516     case ESTR_PROBLEM:
 517     case ESTR_FAILURE:
 518     default:
 519         ch = '.';
 520         break;
 521     }
 522
 523     return ch;
 524 }
 525
 526 /* --------------------------------------------------------------------------------------------- */
 527
 528 int
 529 convert_from_8bit_to_utf_c2 (char input_char)
 530 {
 531     unsigned char str[2];
 532     int ch = '.';
 533     GIConv conv;
 534     const char *cp_from;
 535
 536     str[0] = (unsigned char) input_char;
 537     str[1] = '\0';
 538
 539     cp_from = get_codepage_id (mc_global.source_codepage);
 540     conv = str_crt_conv_to (cp_from);
 541
 542     if (conv != INVALID_CONV)
 543     {
 544         unsigned char buf_ch[UTF8_CHAR_LEN + 1];
 545
 546         switch (str_translate_char (conv, (char *) str, -1, (char *) buf_ch, sizeof (buf_ch)))
 547         {
 548         case ESTR_SUCCESS:
 549             {
 550                 int res;
 551
 552                 res = g_utf8_get_char_validated ((char *) buf_ch, -1);
 553                 ch = res >= 0 ? res : buf_ch[0];
 554                 break;
 555             }
 556         case ESTR_PROBLEM:
 557         case ESTR_FAILURE:
 558         default:
 559             ch = '.';
 560             break;
 561         }
 562         str_close_conv (conv);
 563     }
 564
 565     return ch;
 566 }
 567
 568 /* --------------------------------------------------------------------------------------------- */