2 Unix SMB/CIFS implementation.
3 Character set conversion Extensions
4 Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
5 Copyright (C) Andrew Tridgell 2001
6 Copyright (C) Simo Sorce 2001
7 Copyright (C) Jelmer Vernooij 2007
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 3 of the License, or
12 (at your option) any later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program. If not, see <http://www.gnu.org/licenses/>.
24 #include "system/iconv.h"
29 * @brief Character-set conversion routines built on our iconv.
31 * @note Samba's internal character set (at least in the 3.0 series)
32 * is always the same as the one for the Unix filesystem. It is
33 * <b>not</b> necessarily UTF-8 and may be different on machines that
34 * need i18n filenames to be compatible with Unix software. It does
35 * have to be a superset of ASCII. All multibyte sequences must start
36 * with a byte with the high bit set.
41 struct smb_iconv_convenience
{
42 TALLOC_CTX
*child_ctx
;
43 const char *unix_charset
;
44 const char *dos_charset
;
46 smb_iconv_t conv_handles
[NUM_CHARSETS
][NUM_CHARSETS
];
51 * Return the name of a charset to give to iconv().
53 static const char *charset_name(struct smb_iconv_convenience
*ic
, charset_t ch
)
56 case CH_UTF16
: return "UTF-16LE";
57 case CH_UNIX
: return ic
->unix_charset
;
58 case CH_DOS
: return ic
->dos_charset
;
59 case CH_UTF8
: return "UTF8";
60 case CH_UTF16BE
: return "UTF-16BE";
61 case CH_UTF16MUNGED
: return "UTF16_MUNGED";
68 re-initialize iconv conversion descriptors
70 static int close_iconv_convenience(struct smb_iconv_convenience
*data
)
73 for (c1
=0;c1
<NUM_CHARSETS
;c1
++) {
74 for (c2
=0;c2
<NUM_CHARSETS
;c2
++) {
75 if (data
->conv_handles
[c1
][c2
] != NULL
) {
76 if (data
->conv_handles
[c1
][c2
] != (smb_iconv_t
)-1) {
77 smb_iconv_close(data
->conv_handles
[c1
][c2
]);
79 data
->conv_handles
[c1
][c2
] = NULL
;
88 the old_ic is passed in here as the smb_iconv_convenience structure
89 is used as a global pointer in some places (eg. python modules). We
90 don't want to invalidate those global pointers, but we do want to
91 update them with the right charset information when loadparm
92 runs. To do that we need to re-use the structure pointer, but
93 re-fill the elements in the structure with the updated values
95 _PUBLIC_
struct smb_iconv_convenience
*smb_iconv_convenience_reinit(TALLOC_CTX
*mem_ctx
,
96 const char *dos_charset
,
97 const char *unix_charset
,
99 struct smb_iconv_convenience
*old_ic
)
101 struct smb_iconv_convenience
*ret
;
103 if (old_ic
!= NULL
) {
105 close_iconv_convenience(ret
);
106 talloc_free(ret
->child_ctx
);
109 ret
= talloc_zero(mem_ctx
, struct smb_iconv_convenience
);
115 /* we use a child context to allow us to free all ptrs without
116 freeing the structure itself */
117 ret
->child_ctx
= talloc_new(ret
);
118 if (ret
->child_ctx
== NULL
) {
122 talloc_set_destructor(ret
, close_iconv_convenience
);
124 ret
->dos_charset
= talloc_strdup(ret
->child_ctx
, dos_charset
);
125 ret
->unix_charset
= talloc_strdup(ret
->child_ctx
, unix_charset
);
126 ret
->native_iconv
= native_iconv
;
132 on-demand initialisation of conversion handles
134 static smb_iconv_t
get_conv_handle(struct smb_iconv_convenience
*ic
,
135 charset_t from
, charset_t to
)
138 static bool initialised
;
140 if (initialised
== false) {
144 /* we set back the locale to C to get ASCII-compatible
145 toupper/lower functions. For now we do not need
146 any other POSIX localisations anyway. When we
147 should really need localized string functions one
148 day we need to write our own ascii_tolower etc.
150 setlocale(LC_ALL
, "C");
154 if (ic
->conv_handles
[from
][to
]) {
155 return ic
->conv_handles
[from
][to
];
158 n1
= charset_name(ic
, from
);
159 n2
= charset_name(ic
, to
);
161 ic
->conv_handles
[from
][to
] = smb_iconv_open_ex(ic
, n2
, n1
,
164 if (ic
->conv_handles
[from
][to
] == (smb_iconv_t
)-1) {
165 if ((from
== CH_DOS
|| to
== CH_DOS
) &&
166 strcasecmp(charset_name(ic
, CH_DOS
), "ASCII") != 0) {
167 DEBUG(0,("dos charset '%s' unavailable - using ASCII\n",
168 charset_name(ic
, CH_DOS
)));
169 ic
->dos_charset
= "ASCII";
171 n1
= charset_name(ic
, from
);
172 n2
= charset_name(ic
, to
);
174 ic
->conv_handles
[from
][to
] =
175 smb_iconv_open_ex(ic
, n2
, n1
, ic
->native_iconv
);
179 return ic
->conv_handles
[from
][to
];
183 * Convert string from one encoding to another, making error checking etc
185 * @param mem_ctx Memory context
186 * @param cd Iconv handle
187 * @param src pointer to source string (multibyte or singlebyte)
188 * @param srclen length of the source string in bytes
189 * @param dest pointer to destination string (multibyte or singlebyte)
190 * @param destlen maximal length allowed for string
191 * @returns the number of bytes occupied in the destination
193 _PUBLIC_ ssize_t
iconv_talloc(TALLOC_CTX
*ctx
,
195 void const *src
, size_t srclen
,
198 size_t i_len
, o_len
, destlen
;
199 void **dest
= (void **)dst
;
201 const char *inbuf
= (const char *)src
;
206 /* it is _very_ rare that a conversion increases the size by
211 destlen
= 2 + (destlen
*3);
212 ob
= talloc_realloc(ctx
, outbuf
, char, destlen
);
214 DEBUG(0, ("iconv_talloc: realloc failed!\n"));
221 /* we give iconv 2 less bytes to allow us to terminate at the
225 retval
= smb_iconv(cd
,
228 if(retval
== (size_t)-1) {
229 const char *reason
="unknown error";
232 reason
="Incomplete multibyte sequence";
237 reason
="Illegal multibyte sequence";
240 DEBUG(0,("Conversion error: %s - ",reason
));
241 dump_data(0, (const uint8_t *) inbuf
, i_len
);
246 destlen
= (destlen
-2) - o_len
;
248 /* guarantee null termination in all charsets */
249 SSVAL(ob
, destlen
, 0);
258 * Convert string from one encoding to another, making error checking etc
260 * @param src pointer to source string (multibyte or singlebyte)
261 * @param srclen length of the source string in bytes
262 * @param dest pointer to destination string (multibyte or singlebyte)
263 * @param destlen maximal length allowed for string
264 * @returns the number of bytes occupied in the destination
266 _PUBLIC_
bool convert_string_convenience(struct smb_iconv_convenience
*ic
,
267 charset_t from
, charset_t to
,
268 void const *src
, size_t srclen
,
269 void *dest
, size_t destlen
, size_t *converted_size
,
270 bool allow_badcharcnv
)
274 const char* inbuf
= (const char*)src
;
275 char* outbuf
= (char*)dest
;
276 smb_iconv_t descriptor
;
278 if (allow_badcharcnv
) {
279 /* Not implemented yet */
283 if (srclen
== (size_t)-1)
284 srclen
= strlen(inbuf
)+1;
286 descriptor
= get_conv_handle(ic
, from
, to
);
288 if (descriptor
== (smb_iconv_t
)-1 || descriptor
== (smb_iconv_t
)0) {
289 /* conversion not supported, use as is */
290 size_t len
= MIN(srclen
,destlen
);
291 memcpy(dest
,src
,len
);
292 *converted_size
= len
;
298 retval
= smb_iconv(descriptor
, &inbuf
, &i_len
, &outbuf
, &o_len
);
299 if(retval
==(size_t)-1) {
303 reason
="Incomplete multibyte sequence";
306 reason
="No more room";
307 if (from
== CH_UNIX
) {
308 DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d - '%s'\n",
309 charset_name(ic
, from
), charset_name(ic
, to
),
310 (int)srclen
, (int)destlen
,
313 DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d\n",
314 charset_name(ic
, from
), charset_name(ic
, to
),
315 (int)srclen
, (int)destlen
));
319 reason
="Illegal multibyte sequence";
322 /* smb_panic(reason); */
324 if (converted_size
!= NULL
)
325 *converted_size
= destlen
-o_len
;
330 * Convert between character sets, allocating a new buffer using talloc for the result.
332 * @param srclen length of source buffer.
333 * @param dest always set at least to NULL
334 * @note -1 is not accepted for srclen.
336 * @returns Size in bytes of the converted string; or -1 in case of error.
339 _PUBLIC_
bool convert_string_talloc_convenience(TALLOC_CTX
*ctx
,
340 struct smb_iconv_convenience
*ic
,
341 charset_t from
, charset_t to
,
342 void const *src
, size_t srclen
,
343 void *dst
, size_t *converted_size
,
344 bool allow_badcharcnv
)
346 void **dest
= (void **)dst
;
347 smb_iconv_t descriptor
;
350 if (allow_badcharcnv
)
351 return false; /* Not implemented yet */
355 if (src
== NULL
|| srclen
== (size_t)-1 || srclen
== 0)
358 descriptor
= get_conv_handle(ic
, from
, to
);
360 if (descriptor
== (smb_iconv_t
)-1 || descriptor
== (smb_iconv_t
)0) {
361 /* conversion not supported, return -1*/
362 DEBUG(3, ("convert_string_talloc: conversion from %s to %s not supported!\n",
363 charset_name(ic
, from
),
364 charset_name(ic
, to
)));
368 ret
= iconv_talloc(ctx
, descriptor
, src
, srclen
, dest
);
371 if (converted_size
!= NULL
)
372 *converted_size
= ret
;
377 return the unicode codepoint for the next multi-byte CH_UNIX character
380 also return the number of bytes consumed (which tells the caller
381 how many bytes to skip to get to the next CH_UNIX character)
383 return INVALID_CODEPOINT if the next character cannot be converted
385 _PUBLIC_ codepoint_t
next_codepoint_convenience(struct smb_iconv_convenience
*ic
,
386 const char *str
, size_t *size
)
388 /* it cannot occupy more than 4 bytes in UTF16 format */
390 smb_iconv_t descriptor
;
396 if ((str
[0] & 0x80) == 0) {
398 return (codepoint_t
)str
[0];
401 /* we assume that no multi-byte character can take
402 more than 5 bytes. This is OK as we only
403 support codepoints up to 1M */
404 ilen_orig
= strnlen(str
, 5);
407 descriptor
= get_conv_handle(ic
, CH_UNIX
, CH_UTF16
);
408 if (descriptor
== (smb_iconv_t
)-1) {
410 return INVALID_CODEPOINT
;
413 /* this looks a little strange, but it is needed to cope
414 with codepoints above 64k */
416 outbuf
= (char *)buf
;
417 smb_iconv(descriptor
, &str
, &ilen
, &outbuf
, &olen
);
420 outbuf
= (char *)buf
;
421 smb_iconv(descriptor
, &str
, &ilen
, &outbuf
, &olen
);
423 /* we didn't convert any bytes */
425 return INVALID_CODEPOINT
;
432 *size
= ilen_orig
- ilen
;
435 return (codepoint_t
)SVAL(buf
, 0);
438 /* decode a 4 byte UTF16 character manually */
439 return (codepoint_t
)0x10000 +
440 (buf
[2] | ((buf
[3] & 0x3)<<8) |
441 (buf
[0]<<10) | ((buf
[1] & 0x3)<<18));
444 /* no other length is valid */
445 return INVALID_CODEPOINT
;
449 push a single codepoint into a CH_UNIX string the target string must
450 be able to hold the full character, which is guaranteed if it is at
451 least 5 bytes in size. The caller may pass less than 5 bytes if they
452 are sure the character will fit (for example, you can assume that
453 uppercase/lowercase of a character will not add more than 1 byte)
455 return the number of bytes occupied by the CH_UNIX character, or
458 _PUBLIC_ ssize_t
push_codepoint_convenience(struct smb_iconv_convenience
*ic
,
459 char *str
, codepoint_t c
)
461 smb_iconv_t descriptor
;
471 descriptor
= get_conv_handle(ic
,
473 if (descriptor
== (smb_iconv_t
)-1) {
482 smb_iconv(descriptor
, &inbuf
, &ilen
, &str
, &olen
);
491 buf
[0] = (c
>>10) & 0xFF;
492 buf
[1] = (c
>>18) | 0xd8;
494 buf
[3] = ((c
>>8) & 0x3) | 0xdc;
500 smb_iconv(descriptor
, &inbuf
, &ilen
, &str
, &olen
);