src/os_mac_conv.c

   1 /* vi:set ts=8 sts=4 sw=4:
   2  *
   3  * VIM - Vi IMproved    by Bram Moolenaar
   4  *
   5  * Do ":help uganda"  in Vim to read copying and usage conditions.
   6  * Do ":help credits" in Vim to see a list of people who contributed.
   7  * See README.txt for an overview of the Vim source code.
   8  */
   9 /*
  10  * os_mac_conv.c: Code specifically for Mac string conversions.
  11  *
  12  * This code has been put in a separate file to avoid the conflicts that are
  13  * caused by including both the X11 and Carbon header files.
  14  */
  15
  16 #define NO_X11_INCLUDES
  17 #include "vim.h"
  18
  19 #if defined(MACOS_CONVERT) || defined(PROTO)
  20 # ifdef PROTO
  21 /* A few dummy types to be able to generate function prototypes. */
  22 typedef int UniChar;
  23 typedef int *TECObjectRef;
  24 typedef int CFStringRef;
  25 # endif
  26
  27 static char_u       *mac_utf16_to_utf8 __ARGS((UniChar *from, size_t fromLen, size_t *actualLen));
  28 static UniChar      *mac_utf8_to_utf16 __ARGS((char_u *from, size_t fromLen, size_t *actualLen));
  29
  30 /* Converter for composing decomposed HFS+ file paths */
  31 static TECObjectRef gPathConverter;
  32 /* Converter used by mac_utf16_to_utf8 */
  33 static TECObjectRef gUTF16ToUTF8Converter;
  34
  35 /*
  36  * A Mac version of string_convert_ext() for special cases.
  37  */
  38     char_u *
  39 mac_string_convert(ptr, len, lenp, fail_on_error, from_enc, to_enc, unconvlenp)
  40     char_u              *ptr;
  41     int                 len;
  42     int                 *lenp;
  43     int                 fail_on_error;
  44     int                 from_enc;
  45     int                 to_enc;
  46     int                 *unconvlenp;
  47 {
  48     char_u              *retval, *d;
  49     CFStringRef         cfstr;
  50     int                 buflen, in, out, l, i;
  51     CFStringEncoding    from;
  52     CFStringEncoding    to;
  53
  54     switch (from_enc)
  55     {
  56         case 'l':   from = kCFStringEncodingISOLatin1; break;
  57         case 'm':   from = kCFStringEncodingMacRoman; break;
  58         case 'u':   from = kCFStringEncodingUTF8; break;
  59         default:    return NULL;
  60     }
  61     switch (to_enc)
  62     {
  63         case 'l':   to = kCFStringEncodingISOLatin1; break;
  64         case 'm':   to = kCFStringEncodingMacRoman; break;
  65         case 'u':   to = kCFStringEncodingUTF8; break;
  66         default:    return NULL;
  67     }
  68
  69     if (unconvlenp != NULL)
  70         *unconvlenp = 0;
  71     cfstr = CFStringCreateWithBytes(NULL, ptr, len, from, 0);
  72
  73     if(cfstr == NULL)
  74         fprintf(stderr, "Encoding failed\n");
  75     /* When conversion failed, try excluding bytes from the end, helps when
  76      * there is an incomplete byte sequence.  Only do up to 6 bytes to avoid
  77      * looping a long time when there really is something unconvertible. */
  78     while (cfstr == NULL && unconvlenp != NULL && len > 1 && *unconvlenp < 6)
  79     {
  80         --len;
  81         ++*unconvlenp;
  82         cfstr = CFStringCreateWithBytes(NULL, ptr, len, from, 0);
  83     }
  84     if (cfstr == NULL)
  85         return NULL;
  86
  87     if (to == kCFStringEncodingUTF8)
  88         buflen = len * 6 + 1;
  89     else
  90         buflen = len + 1;
  91     retval = alloc(buflen);
  92     if (retval == NULL)
  93     {
  94         CFRelease(cfstr);
  95         return NULL;
  96     }
  97
  98 #if 0
  99     CFRange convertRange = CFRangeMake(0, CFStringGetLength(cfstr));
 100     /*  Determine output buffer size */
 101     CFStringGetBytes(cfstr, convertRange, to, NULL, FALSE, NULL, 0, (CFIndex *)&buflen);
 102     retval = (buflen > 0) ? alloc(buflen) : NULL;
 103     if (retval == NULL) {
 104         CFRelease(cfstr);
 105         return NULL;
 106     }
 107
 108     if (lenp)
 109         *lenp = buflen / sizeof(char_u);
 110
 111     if (!CFStringGetBytes(cfstr, convertRange, to, NULL, FALSE, retval, buflen, NULL))
 112 #endif
 113     if (!CFStringGetCString(cfstr, (char *)retval, buflen, to))
 114     {
 115         CFRelease(cfstr);
 116         if (fail_on_error)
 117         {
 118             vim_free(retval);
 119             return NULL;
 120         }
 121
 122         fprintf(stderr, "Trying char-by-char conversion...\n");
 123         /* conversion failed for the whole string, but maybe it will work
 124          * for each character */
 125         for (d = retval, in = 0, out = 0; in < len && out < buflen - 1;)
 126         {
 127             if (from == kCFStringEncodingUTF8)
 128                 l = utf_ptr2len(ptr + in);
 129             else
 130                 l = 1;
 131             cfstr = CFStringCreateWithBytes(NULL, ptr + in, l, from, 0);
 132             if (cfstr == NULL)
 133             {
 134                 *d++ = '?';
 135                 out++;
 136             }
 137             else
 138             {
 139                 if (!CFStringGetCString(cfstr, (char *)d, buflen - out, to))
 140                 {
 141                     *d++ = '?';
 142                     out++;
 143                 }
 144                 else
 145                 {
 146                     i = STRLEN(d);
 147                     d += i;
 148                     out += i;
 149                 }
 150                 CFRelease(cfstr);
 151             }
 152             in += l;
 153         }
 154         *d = NUL;
 155         if (lenp != NULL)
 156             *lenp = out;
 157         return retval;
 158     }
 159     CFRelease(cfstr);
 160     if (lenp != NULL)
 161         *lenp = STRLEN(retval);
 162
 163     return retval;
 164 }
 165
 166 /*
 167  * Conversion from Apple MacRoman char encoding to UTF-8 or latin1, using
 168  * standard Carbon framework.
 169  * Input: "ptr[*sizep]".
 170  * "real_size" is the size of the buffer that "ptr" points to.
 171  * output is in-place, "sizep" is adjusted.
 172  * Returns OK or FAIL.
 173  */
 174     int
 175 macroman2enc(ptr, sizep, real_size)
 176     char_u      *ptr;
 177     long        *sizep;
 178     long        real_size;
 179 {
 180     CFStringRef         cfstr;
 181     CFRange             r;
 182     CFIndex             len = *sizep;
 183
 184     /* MacRoman is an 8-bit encoding, no need to move bytes to
 185      * conv_rest[]. */
 186     cfstr = CFStringCreateWithBytes(NULL, ptr, len,
 187                                                 kCFStringEncodingMacRoman, 0);
 188     /*
 189      * If there is a conversion error, try using another
 190      * conversion.
 191      */
 192     if (cfstr == NULL)
 193         return FAIL;
 194
 195     r.location = 0;
 196     r.length = CFStringGetLength(cfstr);
 197     if (r.length != CFStringGetBytes(cfstr, r,
 198             (enc_utf8) ? kCFStringEncodingUTF8 : kCFStringEncodingISOLatin1,
 199             0, /* no lossy conversion */
 200             0, /* not external representation */
 201             ptr + *sizep, real_size - *sizep, &len))
 202     {
 203         CFRelease(cfstr);
 204         return FAIL;
 205     }
 206     CFRelease(cfstr);
 207     mch_memmove(ptr, ptr + *sizep, len);
 208     *sizep = len;
 209
 210     return OK;
 211 }
 212
 213 /*
 214  * Conversion from UTF-8 or latin1 to MacRoman.
 215  * Input: "from[fromlen]"
 216  * Output: "to[maxtolen]" length in "*tolenp"
 217  * Unconverted rest in rest[*restlenp].
 218  * Returns OK or FAIL.
 219  */
 220     int
 221 enc2macroman(from, fromlen, to, tolenp, maxtolen, rest, restlenp)
 222     char_u      *from;
 223     size_t      fromlen;
 224     char_u      *to;
 225     int         *tolenp;
 226     int         maxtolen;
 227     char_u      *rest;
 228     int         *restlenp;
 229 {
 230     CFStringRef cfstr;
 231     CFRange     r;
 232     CFIndex     l;
 233
 234     *restlenp = 0;
 235     cfstr = CFStringCreateWithBytes(NULL, from, fromlen,
 236             (enc_utf8) ? kCFStringEncodingUTF8 : kCFStringEncodingISOLatin1,
 237             0);
 238     while (cfstr == NULL && *restlenp < 3 && fromlen > 1)
 239     {
 240         rest[*restlenp++] = from[--fromlen];
 241         cfstr = CFStringCreateWithBytes(NULL, from, fromlen,
 242                 (enc_utf8) ? kCFStringEncodingUTF8 : kCFStringEncodingISOLatin1,
 243                 0);
 244     }
 245     if (cfstr == NULL)
 246         return FAIL;
 247
 248     r.location = 0;
 249     r.length = CFStringGetLength(cfstr);
 250     if (r.length != CFStringGetBytes(cfstr, r,
 251                 kCFStringEncodingMacRoman,
 252                 0, /* no lossy conversion */
 253                 0, /* not external representation (since vim
 254                     * handles this internally */
 255                 to, maxtolen, &l))
 256     {
 257         CFRelease(cfstr);
 258         return FAIL;
 259     }
 260     CFRelease(cfstr);
 261     *tolenp = l;
 262     return OK;
 263 }
 264
 265 /*
 266  * Initializes text converters
 267  */
 268     void
 269 mac_conv_init()
 270 {
 271     TextEncoding    utf8_encoding;
 272     TextEncoding    utf8_hfsplus_encoding;
 273     TextEncoding    utf8_canon_encoding;
 274     TextEncoding    utf16_encoding;
 275
 276     utf8_encoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
 277             kTextEncodingDefaultVariant, kUnicodeUTF8Format);
 278     utf8_hfsplus_encoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
 279             kUnicodeHFSPlusCompVariant, kUnicodeUTF8Format);
 280     utf8_canon_encoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
 281             kUnicodeCanonicalCompVariant, kUnicodeUTF8Format);
 282     utf16_encoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
 283             kTextEncodingDefaultVariant, kUnicode16BitFormat);
 284
 285     if (TECCreateConverter(&gPathConverter, utf8_encoding,
 286                 utf8_hfsplus_encoding) != noErr)
 287         gPathConverter = NULL;
 288
 289     if (TECCreateConverter(&gUTF16ToUTF8Converter, utf16_encoding,
 290                 utf8_canon_encoding) != noErr)
 291     {
 292         /* On pre-10.3, Unicode normalization is not available so
 293          * fall back to non-normalizing converter */
 294         if (TECCreateConverter(&gUTF16ToUTF8Converter, utf16_encoding,
 295                     utf8_encoding) != noErr)
 296             gUTF16ToUTF8Converter = NULL;
 297     }
 298 }
 299
 300 /*
 301  * Destroys text converters
 302  */
 303     void
 304 mac_conv_cleanup()
 305 {
 306     if (gUTF16ToUTF8Converter)
 307     {
 308         TECDisposeConverter(gUTF16ToUTF8Converter);
 309         gUTF16ToUTF8Converter = NULL;
 310     }
 311
 312     if (gPathConverter)
 313     {
 314         TECDisposeConverter(gPathConverter);
 315         gPathConverter = NULL;
 316     }
 317 }
 318
 319 /*
 320  * Conversion from UTF-16 UniChars to 'encoding'
 321  * The function signature uses the real type of UniChar (as typedef'ed in
 322  * CFBase.h) to avoid clashes with X11 header files in the .pro file
 323  */
 324     char_u *
 325 mac_utf16_to_enc(from, fromLen, actualLen)
 326     unsigned short *from;
 327     size_t fromLen;
 328     size_t *actualLen;
 329 {
 330     /* Following code borrows somewhat from os_mswin.c */
 331     vimconv_T   conv;
 332     size_t      utf8_len;
 333     char_u      *utf8_str;
 334     char_u      *result = NULL;
 335
 336     /* Convert to utf-8 first, works better with iconv */
 337     utf8_len = 0;
 338     utf8_str = mac_utf16_to_utf8(from, fromLen, &utf8_len);
 339
 340     if (utf8_str)
 341     {
 342         /* We might be called before we have p_enc set up. */
 343         conv.vc_type = CONV_NONE;
 344
 345         /* If encoding (p_enc) is any unicode, it is actually in utf-8 (vim
 346          * internal unicode is always utf-8) so don't convert in such cases */
 347
 348         if ((enc_canon_props(p_enc) & ENC_UNICODE) == 0)
 349             convert_setup(&conv, (char_u *)"utf-8",
 350                     p_enc? p_enc: (char_u *)"macroman");
 351         if (conv.vc_type == CONV_NONE)
 352         {
 353             /* p_enc is utf-8, so we're done. */
 354             result = utf8_str;
 355         }
 356         else
 357         {
 358             result = string_convert(&conv, utf8_str, (int *)&utf8_len);
 359             vim_free(utf8_str);
 360         }
 361
 362         convert_setup(&conv, NULL, NULL);
 363
 364         if (actualLen)
 365             *actualLen = utf8_len;
 366     }
 367     else if (actualLen)
 368         *actualLen = 0;
 369
 370     return result;
 371 }
 372
 373 /*
 374  * Conversion from 'encoding' to UTF-16 UniChars
 375  * The function return uses the real type of UniChar (as typedef'ed in
 376  * CFBase.h) to avoid clashes with X11 header files in the .pro file
 377  */
 378     unsigned short *
 379 mac_enc_to_utf16(from, fromLen, actualLen)
 380     char_u *from;
 381     size_t fromLen;
 382     size_t *actualLen;
 383 {
 384     /* Following code borrows somewhat from os_mswin.c */
 385     vimconv_T   conv;
 386     size_t      utf8_len;
 387     char_u      *utf8_str;
 388     UniChar     *result = NULL;
 389     Boolean     should_free_utf8 = FALSE;
 390
 391     do
 392     {
 393         /* Use MacRoman by default, we might be called before we have p_enc
 394          * set up.  Convert to utf-8 first, works better with iconv().  Does
 395          * nothing if 'encoding' is "utf-8". */
 396         conv.vc_type = CONV_NONE;
 397         if ((enc_canon_props(p_enc) & ENC_UNICODE) == 0 &&
 398                 convert_setup(&conv, p_enc ? p_enc : (char_u *)"macroman",
 399                     (char_u *)"utf-8") == FAIL)
 400             break;
 401
 402         if (conv.vc_type != CONV_NONE)
 403         {
 404             utf8_len = fromLen;
 405             utf8_str = string_convert(&conv, from, (int *)&utf8_len);
 406             should_free_utf8 = TRUE;
 407         }
 408         else
 409         {
 410             utf8_str = from;
 411             utf8_len = fromLen;
 412         }
 413
 414         if (utf8_str == NULL)
 415             break;
 416
 417         convert_setup(&conv, NULL, NULL);
 418
 419         result = mac_utf8_to_utf16(utf8_str, utf8_len, actualLen);
 420
 421         if (should_free_utf8)
 422             vim_free(utf8_str);
 423         return result;
 424     }
 425     while (0);
 426
 427     if (actualLen)
 428         *actualLen = 0;
 429
 430     return result;
 431 }
 432
 433 /*
 434  * Converts from UTF-16 UniChars to CFString
 435  * The void * return type is actually a CFStringRef
 436  */
 437     void *
 438 mac_enc_to_cfstring(from, fromLen)
 439     char_u  *from;
 440     size_t  fromLen;
 441 {
 442     UniChar     *utf16_str;
 443     size_t      utf16_len;
 444     CFStringRef result = NULL;
 445
 446     utf16_str = mac_enc_to_utf16(from, fromLen, &utf16_len);
 447     if (utf16_str)
 448     {
 449         result = CFStringCreateWithCharacters(NULL, utf16_str, utf16_len/sizeof(UniChar));
 450         vim_free(utf16_str);
 451     }
 452
 453     return (void *)result;
 454 }
 455
 456 /*
 457  * Converts a decomposed HFS+ UTF-8 path to precomposed UTF-8
 458  */
 459     char_u *
 460 mac_precompose_path(decompPath, decompLen, precompLen)
 461     char_u  *decompPath;
 462     size_t  decompLen;
 463     size_t  *precompLen;
 464 {
 465     char_u  *result = NULL;
 466     size_t  actualLen = 0;
 467
 468     if (gPathConverter)
 469     {
 470         result = alloc(decompLen);
 471         if (result)
 472         {
 473             if (TECConvertText(gPathConverter, decompPath,
 474                         decompLen, &decompLen, result,
 475                         decompLen, &actualLen) != noErr)
 476             {
 477                 vim_free(result);
 478                 result = NULL;
 479             }
 480         }
 481     }
 482
 483     if (precompLen)
 484         *precompLen = actualLen;
 485
 486     return result;
 487 }
 488
 489 /*
 490  * Converts from UTF-16 UniChars to precomposed UTF-8
 491  */
 492     static char_u *
 493 mac_utf16_to_utf8(from, fromLen, actualLen)
 494     UniChar *from;
 495     size_t fromLen;
 496     size_t *actualLen;
 497 {
 498     ByteCount           utf8_len;
 499     ByteCount           inputRead;
 500     char_u              *result;
 501
 502     if (gUTF16ToUTF8Converter)
 503     {
 504         result = alloc(fromLen * 6 + 1);
 505         if (result && TECConvertText(gUTF16ToUTF8Converter, (ConstTextPtr)from,
 506                     fromLen, &inputRead, result,
 507                     (fromLen*6+1)*sizeof(char_u), &utf8_len) == noErr)
 508         {
 509             TECFlushText(gUTF16ToUTF8Converter, result, (fromLen*6+1)*sizeof(char_u), &inputRead);
 510             utf8_len += inputRead;
 511         }
 512         else
 513         {
 514             vim_free(result);
 515             result = NULL;
 516         }
 517     }
 518     else
 519     {
 520         result = NULL;
 521     }
 522
 523     if (actualLen)
 524         *actualLen = result ? utf8_len : 0;
 525
 526     return result;
 527 }
 528
 529 /*
 530  * Converts from UTF-8 to UTF-16 UniChars
 531  */
 532     static UniChar *
 533 mac_utf8_to_utf16(from, fromLen, actualLen)
 534     char_u *from;
 535     size_t fromLen;
 536     size_t *actualLen;
 537 {
 538     CFStringRef  utf8_str;
 539     CFRange      convertRange;
 540     UniChar      *result = NULL;
 541
 542     utf8_str = CFStringCreateWithBytes(NULL, from, fromLen,
 543             kCFStringEncodingUTF8, FALSE);
 544
 545     if (utf8_str == NULL) {
 546         if (actualLen)
 547             *actualLen = 0;
 548         return NULL;
 549     }
 550
 551     convertRange = CFRangeMake(0, CFStringGetLength(utf8_str));
 552     result = (UniChar *)alloc(convertRange.length * sizeof(UniChar));
 553
 554     CFStringGetCharacters(utf8_str, convertRange, result);
 555
 556     CFRelease(utf8_str);
 557
 558     if (actualLen)
 559         *actualLen = convertRange.length * sizeof(UniChar);
 560
 561     return result;
 562 }
 563
 564 /*
 565  * Sets LANG environment variable in Vim from Mac locale
 566  */
 567     void
 568 mac_lang_init() {
 569     if (mch_getenv((char_u *)"LANG") == NULL)
 570     {
 571         char    buf[20];
 572         if (LocaleRefGetPartString(NULL,
 573                     kLocaleLanguageMask | kLocaleLanguageVariantMask |
 574                     kLocaleRegionMask | kLocaleRegionVariantMask,
 575                     sizeof buf, buf) == noErr && *buf)
 576         {
 577             vim_setenv((char_u *)"LANG", (char_u *)buf);
 578 #   ifdef HAVE_LOCALE_H
 579             setlocale(LC_ALL, "");
 580 #   endif
 581         }
 582     }
 583 }
 584 #endif /* MACOS_CONVERT */