source/lib/iconv.c

   1 /*
   2    Unix SMB/CIFS implementation.
   3    minimal iconv implementation
   4    Copyright (C) Andrew Tridgell 2001
   5    Copyright (C) Jelmer Vernooij 2002,2003
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 2 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20 */
  21
  22 #include "includes.h"
  23
  24 /*
  25  * We have to use strcasecmp here as the character conversions
  26  * haven't been initialised yet. JRA.
  27  */
  28
  29 #undef strcasecmp
  30
  31 /**
  32  * @file
  33  *
  34  * @brief Samba wrapper/stub for iconv character set conversion.
  35  *
  36  * iconv is the XPG2 interface for converting between character
  37  * encodings.  This file provides a Samba wrapper around it, and also
  38  * a simple reimplementation that is used if the system does not
  39  * implement iconv.
  40  *
  41  * Samba only works with encodings that are supersets of ASCII: ascii
  42  * characters like whitespace can be tested for directly, multibyte
  43  * sequences start with a byte with the high bit set, and strings are
  44  * terminated by a nul byte.
  45  *
  46  * Note that the only function provided by iconv is conversion between
  47  * characters.  It doesn't directly support operations like
  48  * uppercasing or comparison.  We have to convert to UCS-2 and compare
  49  * there.
  50  *
  51  * @sa Samba Developers Guide
  52  **/
  53
  54 static size_t ascii_pull(void *,char **, size_t *, char **, size_t *);
  55 static size_t ascii_push(void *,char **, size_t *, char **, size_t *);
  56 static size_t latin1_push(void *,char **, size_t *, char **, size_t *);
  57 static size_t  utf8_pull(void *,char **, size_t *, char **, size_t *);
  58 static size_t  utf8_push(void *,char **, size_t *, char **, size_t *);
  59 static size_t ucs2hex_pull(void *,char **, size_t *, char **, size_t *);
  60 static size_t ucs2hex_push(void *,char **, size_t *, char **, size_t *);
  61 static size_t iconv_copy(void *,char **, size_t *, char **, size_t *);
  62
  63 static struct charset_functions builtin_functions[] = {
  64         {"UCS-2LE",  iconv_copy, iconv_copy},
  65         {"UTF8",   utf8_pull,  utf8_push},
  66         {"ASCII", ascii_pull, ascii_push},
  67         {"646", ascii_pull, ascii_push},
  68         {"ISO-8859-1", ascii_pull, latin1_push},
  69         {"UCS2-HEX", ucs2hex_pull, ucs2hex_push},
  70         {NULL, NULL, NULL}
  71 };
  72
  73 static struct charset_functions *charsets = NULL;
  74
  75 static struct charset_functions *find_charset_functions(const char *name)
  76 {
  77         struct charset_functions *c = charsets;
  78
  79         while(c) {
  80                 if (strcasecmp(name, c->name) == 0) {
  81                         return c;
  82                 }
  83                 c = c->next;
  84         }
  85
  86         return NULL;
  87 }
  88
  89 NTSTATUS smb_register_charset(struct charset_functions *funcs)
  90 {
  91         if (!funcs) {
  92                 return NT_STATUS_INVALID_PARAMETER;
  93         }
  94
  95         DEBUG(5, ("Attempting to register new charset %s\n", funcs->name));
  96         /* Check whether we already have this charset... */
  97         if (find_charset_functions(funcs->name)) {
  98                 DEBUG(0, ("Duplicate charset %s, not registering\n", funcs->name));
  99                 return NT_STATUS_OBJECT_NAME_COLLISION;
 100         }
 101
 102         funcs->next = funcs->prev = NULL;
 103         DEBUG(5, ("Registered charset %s\n", funcs->name));
 104         DLIST_ADD(charsets, funcs);
 105         return NT_STATUS_OK;
 106 }
 107
 108 static void lazy_initialize_iconv(void)
 109 {
 110         static BOOL initialized;
 111         int i;
 112
 113         if (!initialized) {
 114                 initialized = True;
 115                 for(i = 0; builtin_functions[i].name; i++)
 116                         smb_register_charset(&builtin_functions[i]);
 117                 static_init_charset;
 118         }
 119 }
 120
 121 /* if there was an error then reset the internal state,
 122    this ensures that we don't have a shift state remaining for
 123    character sets like SJIS */
 124 static size_t sys_iconv(void *cd,
 125                         char **inbuf, size_t *inbytesleft,
 126                         char **outbuf, size_t *outbytesleft)
 127 {
 128 #ifdef HAVE_NATIVE_ICONV
 129         size_t ret = iconv((iconv_t)cd,
 130                            inbuf, inbytesleft,
 131                            outbuf, outbytesleft);
 132         if (ret == (size_t)-1) {
 133                 int saved_errno = errno;
 134                 iconv(cd, NULL, NULL, NULL, NULL);
 135                 errno = saved_errno;
 136         }
 137         return ret;
 138 #else
 139         errno = EINVAL;
 140         return -1;
 141 #endif
 142 }
 143
 144 /**
 145  * This is a simple portable iconv() implementaion.
 146  *
 147  * It only knows about a very small number of character sets - just
 148  * enough that Samba works on systems that don't have iconv.
 149  **/
 150 size_t smb_iconv(smb_iconv_t cd,
 151                  char **inbuf, size_t *inbytesleft,
 152                  char **outbuf, size_t *outbytesleft)
 153 {
 154         char cvtbuf[2048];
 155         char *bufp = cvtbuf;
 156         size_t bufsize;
 157
 158         /* in many cases we can go direct */
 159         if (cd->direct) {
 160                 return cd->direct(cd->cd_direct,
 161                                   (char **)inbuf, inbytesleft, outbuf, outbytesleft);
 162         }
 163
 164
 165         /* otherwise we have to do it chunks at a time */
 166         while (*inbytesleft > 0) {
 167                 bufp = cvtbuf;
 168                 bufsize = sizeof(cvtbuf);
 169
 170                 if (cd->pull(cd->cd_pull,
 171                              (char **)inbuf, inbytesleft, &bufp, &bufsize) == -1
 172                     && errno != E2BIG) return -1;
 173
 174                 bufp = cvtbuf;
 175                 bufsize = sizeof(cvtbuf) - bufsize;
 176
 177                 if (cd->push(cd->cd_push,
 178                              &bufp, &bufsize,
 179                              outbuf, outbytesleft) == -1) return -1;
 180         }
 181
 182         return 0;
 183 }
 184
 185 /*
 186   simple iconv_open() wrapper
 187  */
 188 smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
 189 {
 190         smb_iconv_t ret;
 191         struct charset_functions *from, *to;
 192
 193         lazy_initialize_iconv();
 194         from = charsets;
 195         to = charsets;
 196
 197         ret = (smb_iconv_t)malloc(sizeof(*ret));
 198         if (!ret) {
 199                 errno = ENOMEM;
 200                 return (smb_iconv_t)-1;
 201         }
 202         memset(ret, 0, sizeof(*ret));
 203
 204         ret->from_name = strdup(fromcode);
 205         ret->to_name = strdup(tocode);
 206
 207         /* check for the simplest null conversion */
 208         if (strcasecmp(fromcode, tocode) == 0) {
 209                 ret->direct = iconv_copy;
 210                 return ret;
 211         }
 212
 213         /* check if we have a builtin function for this conversion */
 214         from = find_charset_functions(fromcode);
 215         if(from)ret->pull = from->pull;
 216
 217         to = find_charset_functions(tocode);
 218         if(to)ret->push = to->push;
 219
 220         /* check if we can use iconv for this conversion */
 221 #ifdef HAVE_NATIVE_ICONV
 222         if (!ret->pull) {
 223                 ret->cd_pull = iconv_open("UCS-2LE", fromcode);
 224                 if (ret->cd_pull != (iconv_t)-1)
 225                         ret->pull = sys_iconv;
 226         }
 227
 228         if (!ret->push) {
 229                 ret->cd_push = iconv_open(tocode, "UCS-2LE");
 230                 if (ret->cd_push != (iconv_t)-1)
 231                         ret->push = sys_iconv;
 232         }
 233 #endif
 234
 235         /* check if there is a module available that can do this conversion */
 236         if (!ret->pull && NT_STATUS_IS_OK(smb_probe_module("charset", fromcode))) {
 237                 if(!(from = find_charset_functions(fromcode)))
 238                         DEBUG(0, ("Module %s doesn't provide charset %s!\n", fromcode, fromcode));
 239                 else
 240                         ret->pull = from->pull;
 241         }
 242
 243         if (!ret->push && NT_STATUS_IS_OK(smb_probe_module("charset", tocode))) {
 244                 if(!(to = find_charset_functions(tocode)))
 245                         DEBUG(0, ("Module %s doesn't provide charset %s!\n", tocode, tocode));
 246                 else
 247                         ret->push = to->push;
 248         }
 249
 250         if (!ret->push || !ret->pull) {
 251                 SAFE_FREE(ret->from_name);
 252                 SAFE_FREE(ret->to_name);
 253                 SAFE_FREE(ret);
 254                 errno = EINVAL;
 255                 return (smb_iconv_t)-1;
 256         }
 257
 258         /* check for conversion to/from ucs2 */
 259         if (strcasecmp(fromcode, "UCS-2LE") == 0 && to) {
 260                 ret->direct = to->push;
 261                 ret->push = ret->pull = NULL;
 262                 return ret;
 263         }
 264
 265         if (strcasecmp(tocode, "UCS-2LE") == 0 && from) {
 266                 ret->direct = from->pull;
 267                 ret->push = ret->pull = NULL;
 268                 return ret;
 269         }
 270
 271         /* Check if we can do the conversion direct */
 272 #ifdef HAVE_NATIVE_ICONV
 273         if (strcasecmp(fromcode, "UCS-2LE") == 0) {
 274                 ret->direct = sys_iconv;
 275                 ret->cd_direct = ret->cd_push;
 276                 ret->cd_push = NULL;
 277                 return ret;
 278         }
 279         if (strcasecmp(tocode, "UCS-2LE") == 0) {
 280                 ret->direct = sys_iconv;
 281                 ret->cd_direct = ret->cd_pull;
 282                 ret->cd_pull = NULL;
 283                 return ret;
 284         }
 285 #endif
 286
 287         return ret;
 288 }
 289
 290 /*
 291   simple iconv_close() wrapper
 292 */
 293 int smb_iconv_close (smb_iconv_t cd)
 294 {
 295 #ifdef HAVE_NATIVE_ICONV
 296         if (cd->cd_direct) iconv_close((iconv_t)cd->cd_direct);
 297         if (cd->cd_pull) iconv_close((iconv_t)cd->cd_pull);
 298         if (cd->cd_push) iconv_close((iconv_t)cd->cd_push);
 299 #endif
 300
 301         SAFE_FREE(cd->from_name);
 302         SAFE_FREE(cd->to_name);
 303
 304         memset(cd, 0, sizeof(*cd));
 305         SAFE_FREE(cd);
 306         return 0;
 307 }
 308
 309
 310 /**********************************************************************
 311  the following functions implement the builtin character sets in Samba
 312  and also the "test" character sets that are designed to test
 313  multi-byte character set support for english users
 314 ***********************************************************************/
 315
 316 static size_t ascii_pull(void *cd, char **inbuf, size_t *inbytesleft,
 317                          char **outbuf, size_t *outbytesleft)
 318 {
 319         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 320                 (*outbuf)[0] = (*inbuf)[0];
 321                 (*outbuf)[1] = 0;
 322                 (*inbytesleft)  -= 1;
 323                 (*outbytesleft) -= 2;
 324                 (*inbuf)  += 1;
 325                 (*outbuf) += 2;
 326         }
 327
 328         if (*inbytesleft > 0) {
 329                 errno = E2BIG;
 330                 return -1;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static size_t ascii_push(void *cd, char **inbuf, size_t *inbytesleft,
 337                          char **outbuf, size_t *outbytesleft)
 338 {
 339         int ir_count=0;
 340
 341         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 342                 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
 343                 if ((*inbuf)[1]) ir_count++;
 344                 (*inbytesleft)  -= 2;
 345                 (*outbytesleft) -= 1;
 346                 (*inbuf)  += 2;
 347                 (*outbuf) += 1;
 348         }
 349
 350         if (*inbytesleft == 1) {
 351                 errno = EINVAL;
 352                 return -1;
 353         }
 354
 355         if (*inbytesleft > 1) {
 356                 errno = E2BIG;
 357                 return -1;
 358         }
 359
 360         return ir_count;
 361 }
 362
 363 static size_t latin1_push(void *cd, char **inbuf, size_t *inbytesleft,
 364                          char **outbuf, size_t *outbytesleft)
 365 {
 366         int ir_count=0;
 367
 368         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 369                 (*outbuf)[0] = (*inbuf)[0];
 370                 if ((*inbuf)[1]) ir_count++;
 371                 (*inbytesleft)  -= 2;
 372                 (*outbytesleft) -= 1;
 373                 (*inbuf)  += 2;
 374                 (*outbuf) += 1;
 375         }
 376
 377         if (*inbytesleft == 1) {
 378                 errno = EINVAL;
 379                 return -1;
 380         }
 381
 382         if (*inbytesleft > 1) {
 383                 errno = E2BIG;
 384                 return -1;
 385         }
 386
 387         return ir_count;
 388 }
 389
 390 static size_t ucs2hex_pull(void *cd, char **inbuf, size_t *inbytesleft,
 391                          char **outbuf, size_t *outbytesleft)
 392 {
 393         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 394                 unsigned v;
 395
 396                 if ((*inbuf)[0] != '@') {
 397                         /* seven bit ascii case */
 398                         (*outbuf)[0] = (*inbuf)[0];
 399                         (*outbuf)[1] = 0;
 400                         (*inbytesleft)  -= 1;
 401                         (*outbytesleft) -= 2;
 402                         (*inbuf)  += 1;
 403                         (*outbuf) += 2;
 404                         continue;
 405                 }
 406                 /* it's a hex character */
 407                 if (*inbytesleft < 5) {
 408                         errno = EINVAL;
 409                         return -1;
 410                 }
 411
 412                 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
 413                         errno = EILSEQ;
 414                         return -1;
 415                 }
 416
 417                 (*outbuf)[0] = v&0xff;
 418                 (*outbuf)[1] = v>>8;
 419                 (*inbytesleft)  -= 5;
 420                 (*outbytesleft) -= 2;
 421                 (*inbuf)  += 5;
 422                 (*outbuf) += 2;
 423         }
 424
 425         if (*inbytesleft > 0) {
 426                 errno = E2BIG;
 427                 return -1;
 428         }
 429
 430         return 0;
 431 }
 432
 433 static size_t ucs2hex_push(void *cd, char **inbuf, size_t *inbytesleft,
 434                            char **outbuf, size_t *outbytesleft)
 435 {
 436         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 437                 char buf[6];
 438
 439                 if ((*inbuf)[1] == 0 &&
 440                     ((*inbuf)[0] & 0x80) == 0 &&
 441                     (*inbuf)[0] != '@') {
 442                         (*outbuf)[0] = (*inbuf)[0];
 443                         (*inbytesleft)  -= 2;
 444                         (*outbytesleft) -= 1;
 445                         (*inbuf)  += 2;
 446                         (*outbuf) += 1;
 447                         continue;
 448                 }
 449                 if (*outbytesleft < 5) {
 450                         errno = E2BIG;
 451                         return -1;
 452                 }
 453                 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
 454                 memcpy(*outbuf, buf, 5);
 455                 (*inbytesleft)  -= 2;
 456                 (*outbytesleft) -= 5;
 457                 (*inbuf)  += 2;
 458                 (*outbuf) += 5;
 459         }
 460
 461         if (*inbytesleft == 1) {
 462                 errno = EINVAL;
 463                 return -1;
 464         }
 465
 466         if (*inbytesleft > 1) {
 467                 errno = E2BIG;
 468                 return -1;
 469         }
 470
 471         return 0;
 472 }
 473
 474
 475 static size_t iconv_copy(void *cd, char **inbuf, size_t *inbytesleft,
 476                          char **outbuf, size_t *outbytesleft)
 477 {
 478         int n;
 479
 480         n = MIN(*inbytesleft, *outbytesleft);
 481
 482         memmove(*outbuf, *inbuf, n);
 483
 484         (*inbytesleft) -= n;
 485         (*outbytesleft) -= n;
 486         (*inbuf) += n;
 487         (*outbuf) += n;
 488
 489         if (*inbytesleft > 0) {
 490                 errno = E2BIG;
 491                 return -1;
 492         }
 493
 494         return 0;
 495 }
 496
 497 static size_t utf8_pull(void *cd, char **inbuf, size_t *inbytesleft,
 498                          char **outbuf, size_t *outbytesleft)
 499 {
 500         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 501                 unsigned char *c = (unsigned char *)*inbuf;
 502                 unsigned char *uc = (unsigned char *)*outbuf;
 503                 int len = 1;
 504
 505                 if ((c[0] & 0x80) == 0) {
 506                         uc[0] = c[0];
 507                         uc[1] = 0;
 508                 } else if ((c[0] & 0xf0) == 0xe0) {
 509                         if (*inbytesleft < 3) {
 510                                 DEBUG(0,("short utf8 char\n"));
 511                                 goto badseq;
 512                         }
 513                         uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
 514                         uc[0] = (c[1]<<6) | (c[2]&0x3f);
 515                         len = 3;
 516                 } else if ((c[0] & 0xe0) == 0xc0) {
 517                         if (*inbytesleft < 2) {
 518                                 DEBUG(0,("short utf8 char\n"));
 519                                 goto badseq;
 520                         }
 521                         uc[1] = (c[0]>>2) & 0x7;
 522                         uc[0] = (c[0]<<6) | (c[1]&0x3f);
 523                         len = 2;
 524                 }
 525
 526                 (*inbuf)  += len;
 527                 (*inbytesleft)  -= len;
 528                 (*outbytesleft) -= 2;
 529                 (*outbuf) += 2;
 530         }
 531
 532         if (*inbytesleft > 0) {
 533                 errno = E2BIG;
 534                 return -1;
 535         }
 536
 537         return 0;
 538
 539 badseq:
 540         errno = EINVAL;
 541         return -1;
 542 }
 543
 544 static size_t utf8_push(void *cd, char **inbuf, size_t *inbytesleft,
 545                          char **outbuf, size_t *outbytesleft)
 546 {
 547         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 548                 unsigned char *c = (unsigned char *)*outbuf;
 549                 unsigned char *uc = (unsigned char *)*inbuf;
 550                 int len=1;
 551
 552                 if (uc[1] & 0xf8) {
 553                         if (*outbytesleft < 3) {
 554                                 DEBUG(0,("short utf8 write\n"));
 555                                 goto toobig;
 556                         }
 557                         c[0] = 0xe0 | (uc[1]>>4);
 558                         c[1] = 0x80 | ((uc[1]&0xF)<<2) | (uc[0]>>6);
 559                         c[2] = 0x80 | (uc[0]&0x3f);
 560                         len = 3;
 561                 } else if (uc[1] | (uc[0] & 0x80)) {
 562                         if (*outbytesleft < 2) {
 563                                 DEBUG(0,("short utf8 write\n"));
 564                                 goto toobig;
 565                         }
 566                         c[0] = 0xc0 | (uc[1]<<2) | (uc[0]>>6);
 567                         c[1] = 0x80 | (uc[0]&0x3f);
 568                         len = 2;
 569                 } else {
 570                         c[0] = uc[0];
 571                 }
 572
 573
 574                 (*inbytesleft)  -= 2;
 575                 (*outbytesleft) -= len;
 576                 (*inbuf)  += 2;
 577                 (*outbuf) += len;
 578         }
 579
 580         if (*inbytesleft == 1) {
 581                 errno = EINVAL;
 582                 return -1;
 583         }
 584
 585         if (*inbytesleft > 1) {
 586                 errno = E2BIG;
 587                 return -1;
 588         }
 589
 590         return 0;
 591
 592 toobig:
 593         errno = E2BIG;
 594         return -1;
 595 }