2 Unix SMB/CIFS implementation.
3 Character set conversion Extensions
4 Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
5 Copyright (C) Andrew Tridgell 2001
6 Copyright (C) Simo Sorce 2001
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 * @brief Character-set conversion routines built on our iconv.
30 * @note Samba's internal character set (at least in the 3.0 series)
31 * is always the same as the one for the Unix filesystem. It is
32 * <b>not</b> necessarily UTF-8 and may be different on machines that
33 * need i18n filenames to be compatible with Unix software. It does
34 * have to be a superset of ASCII. All multibyte sequences must start
35 * with a byte with the high bit set.
40 static pstring cvtbuf;
42 static smb_iconv_t conv_handles[NUM_CHARSETS][NUM_CHARSETS];
46 * Return the name of a charset to give to iconv().
48 static const char *charset_name(charset_t ch)
50 const char *ret = NULL;
52 if (ch == CH_UCS2) ret = "UCS-2LE";
53 else if (ch == CH_UNIX) ret = lp_unix_charset();
54 else if (ch == CH_DOS) ret = lp_dos_charset();
55 else if (ch == CH_DISPLAY) ret = lp_display_charset();
56 else if (ch == CH_UTF8) ret = "UTF8";
58 if (!ret || !*ret) ret = "ASCII";
62 static void lazy_initialize_conv(void)
64 static int initialized = False;
75 Initialize iconv conversion descriptors.
81 BOOL did_reload = False;
83 /* so that charset_name() works we need to get the UNIX<->UCS2 going
85 if (!conv_handles[CH_UNIX][CH_UCS2])
86 conv_handles[CH_UNIX][CH_UCS2] = smb_iconv_open("UCS-2LE", "ASCII");
88 if (!conv_handles[CH_UCS2][CH_UNIX])
89 conv_handles[CH_UCS2][CH_UNIX] = smb_iconv_open("ASCII", "UCS-2LE");
91 for (c1=0;c1<NUM_CHARSETS;c1++) {
92 for (c2=0;c2<NUM_CHARSETS;c2++) {
93 const char *n1 = charset_name((charset_t)c1);
94 const char *n2 = charset_name((charset_t)c2);
95 if (conv_handles[c1][c2] &&
96 strcmp(n1, conv_handles[c1][c2]->from_name) == 0 &&
97 strcmp(n2, conv_handles[c1][c2]->to_name) == 0)
102 if (conv_handles[c1][c2])
103 smb_iconv_close(conv_handles[c1][c2]);
105 conv_handles[c1][c2] = smb_iconv_open(n2,n1);
106 if (conv_handles[c1][c2] == (smb_iconv_t)-1) {
107 DEBUG(0,("Conversion from %s to %s not supported\n",
108 charset_name((charset_t)c1), charset_name((charset_t)c2)));
109 conv_handles[c1][c2] = NULL;
120 * Convert string from one encoding to another, making error checking etc
122 * @param src pointer to source string (multibyte or singlebyte)
123 * @param srclen length of the source string in bytes
124 * @param dest pointer to destination string (multibyte or singlebyte)
125 * @param destlen maximal length allowed for string
126 * @returns the number of bytes occupied in the destination
128 size_t convert_string(charset_t from, charset_t to,
129 void const *src, size_t srclen,
130 void *dest, size_t destlen)
134 const char* inbuf = (const char*)src;
135 char* outbuf = (char*)dest;
136 smb_iconv_t descriptor;
138 if (srclen == (size_t)-1)
139 srclen = strlen(src)+1;
141 lazy_initialize_conv();
143 descriptor = conv_handles[from][to];
145 if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
146 /* conversion not supported, use as is */
147 size_t len = MIN(srclen,destlen);
148 memcpy(dest,src,len);
154 retval = smb_iconv(descriptor, &inbuf, &i_len, &outbuf, &o_len);
155 if(retval==(size_t)-1) {
156 const char *reason="unknown error";
159 reason="Incomplete multibyte sequence";
162 reason="No more room";
163 DEBUG(0, ("convert_string: Required %d, available %d\n",
165 /* we are not sure we need srclen bytes,
166 may be more, may be less.
167 We only know we need more than destlen
171 reason="Illegal multibyte sequence";
174 /* smb_panic(reason); */
176 return destlen-o_len;
180 * Convert between character sets, allocating a new buffer for the result.
182 * @param srclen length of source buffer.
183 * @param dest always set at least to NULL
184 * @note -1 is not accepted for srclen.
186 * @returns Size in bytes of the converted string; or -1 in case of error.
189 static size_t convert_string_allocate(charset_t from, charset_t to,
190 void const *src, size_t srclen, void **dest)
192 size_t i_len, o_len, destlen;
194 const char *inbuf = (const char *)src;
196 smb_iconv_t descriptor;
200 if (src == NULL || srclen == (size_t)-1)
203 lazy_initialize_conv();
205 descriptor = conv_handles[from][to];
207 if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
208 /* conversion not supported, return -1*/
209 DEBUG(3, ("convert_string_allocate: conversion not supported!\n"));
213 destlen = MAX(srclen, 512);
216 destlen = destlen * 2;
217 ob = (char *)realloc(outbuf, destlen);
219 DEBUG(0, ("convert_string_allocate: realloc failed!\n"));
227 retval = smb_iconv(descriptor,
230 if(retval == (size_t)-1) {
231 const char *reason="unknown error";
234 reason="Incomplete multibyte sequence";
239 reason="Illegal multibyte sequence";
242 DEBUG(0,("Conversion error: %s(%s)\n",reason,inbuf));
243 /* smb_panic(reason); */
247 destlen = destlen - o_len;
248 *dest = (char *)Realloc(ob,destlen);
250 DEBUG(0, ("convert_string_allocate: out of memory!\n"));
260 * Convert between character sets, allocating a new buffer using talloc for the result.
262 * @param srclen length of source buffer.
263 * @param dest always set at least to NULL
264 * @note -1 is not accepted for srclen.
266 * @returns Size in bytes of the converted string; or -1 in case of error.
268 static size_t convert_string_talloc(TALLOC_CTX *ctx, charset_t from, charset_t to,
269 void const *src, size_t srclen, void **dest)
271 void *alloced_string;
275 dest_len=convert_string_allocate(from, to, src, srclen, &alloced_string);
276 if (dest_len == (size_t)-1)
278 *dest = talloc_memdup(ctx, alloced_string, dest_len);
279 SAFE_FREE(alloced_string);
285 size_t unix_strupper(const char *src, size_t srclen, char *dest, size_t destlen)
288 smb_ucs2_t *buffer=(smb_ucs2_t*)cvtbuf;
289 size=convert_string(CH_UNIX, CH_UCS2, src, srclen, buffer, sizeof(cvtbuf));
290 if (!strupper_w(buffer) && (dest == src))
292 return convert_string(CH_UCS2, CH_UNIX, buffer, size, dest, destlen);
295 size_t unix_strlower(const char *src, size_t srclen, char *dest, size_t destlen)
298 smb_ucs2_t *buffer=(smb_ucs2_t*)cvtbuf;
299 size=convert_string(CH_UNIX, CH_UCS2, src, srclen, buffer, sizeof(cvtbuf));
300 if (!strlower_w(buffer) && (dest == src))
302 return convert_string(CH_UCS2, CH_UNIX, buffer, size, dest, destlen);
306 static size_t ucs2_align(const void *base_ptr, const void *p, int flags)
308 if (flags & (STR_NOALIGN|STR_ASCII))
310 return PTR_DIFF(p, base_ptr) & 1;
315 * Copy a string from a char* unix src to a dos codepage string destination.
317 * @return the number of bytes occupied by the string in the destination.
319 * @param flags can include
321 * <dt>STR_TERMINATE</dt> <dd>means include the null termination</dd>
322 * <dt>STR_UPPER</dt> <dd>means uppercase in the destination</dd>
325 * @param dest_len the maximum length in bytes allowed in the
326 * destination. If @p dest_len is -1 then no maximum is used.
328 size_t push_ascii(void *dest, const char *src, size_t dest_len, int flags)
330 size_t src_len = strlen(src);
333 /* treat a pstring as "unlimited" length */
334 if (dest_len == (size_t)-1)
335 dest_len = sizeof(pstring);
337 if (flags & STR_UPPER) {
338 pstrcpy(tmpbuf, src);
343 if (flags & STR_TERMINATE)
346 return convert_string(CH_UNIX, CH_DOS, src, src_len, dest, dest_len);
349 size_t push_ascii_fstring(void *dest, const char *src)
351 return push_ascii(dest, src, sizeof(fstring), STR_TERMINATE);
354 size_t push_ascii_pstring(void *dest, const char *src)
356 return push_ascii(dest, src, sizeof(pstring), STR_TERMINATE);
360 * Copy a string from a dos codepage source to a unix char* destination.
362 * The resulting string in "dest" is always null terminated.
364 * @param flags can have:
366 * <dt>STR_TERMINATE</dt>
367 * <dd>STR_TERMINATE means the string in @p src
368 * is null terminated, and src_len is ignored.</dd>
371 * @param src_len is the length of the source area in bytes.
372 * @returns the number of bytes occupied by the string in @p src.
374 size_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
378 if (dest_len == (size_t)-1)
379 dest_len = sizeof(pstring);
381 if (flags & STR_TERMINATE) {
382 if (src_len == (size_t)-1) {
383 src_len = strlen(src) + 1;
385 size_t len = strnlen(src, src_len);
392 ret = convert_string(CH_DOS, CH_UNIX, src, src_len, dest, dest_len);
395 dest[MIN(ret, dest_len-1)] = 0;
400 size_t pull_ascii_pstring(char *dest, const void *src)
402 return pull_ascii(dest, src, sizeof(pstring), -1, STR_TERMINATE);
405 size_t pull_ascii_fstring(char *dest, const void *src)
407 return pull_ascii(dest, src, sizeof(fstring), -1, STR_TERMINATE);
411 * Copy a string from a char* src to a unicode destination.
413 * @returns the number of bytes occupied by the string in the destination.
415 * @param flags can have:
418 * <dt>STR_TERMINATE <dd>means include the null termination.
419 * <dt>STR_UPPER <dd>means uppercase in the destination.
420 * <dt>STR_NOALIGN <dd>means don't do alignment.
423 * @param dest_len is the maximum length allowed in the
424 * destination. If dest_len is -1 then no maxiumum is used.
426 size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
429 size_t src_len = strlen(src);
432 /* treat a pstring as "unlimited" length */
433 if (dest_len == (size_t)-1)
434 dest_len = sizeof(pstring);
436 if (flags & STR_UPPER) {
437 pstrcpy(tmpbuf, src);
442 if (flags & STR_TERMINATE)
445 if (ucs2_align(base_ptr, dest, flags)) {
447 dest = (void *)((char *)dest + 1);
448 if (dest_len) dest_len--;
452 /* ucs2 is always a multiple of 2 bytes */
455 len += convert_string(CH_UNIX, CH_UCS2, src, src_len, dest, dest_len);
461 * Copy a string from a unix char* src to a UCS2 destination,
462 * allocating a buffer using talloc().
464 * @param dest always set at least to NULL
466 * @returns The number of bytes occupied by the string in the destination
467 * or -1 in case of error.
469 size_t push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src)
471 size_t src_len = strlen(src)+1;
474 return convert_string_talloc(ctx, CH_UNIX, CH_UCS2, src, src_len, (void **)dest);
479 * Copy a string from a unix char* src to a UCS2 destination, allocating a buffer
481 * @param dest always set at least to NULL
483 * @returns The number of bytes occupied by the string in the destination
484 * or -1 in case of error.
487 size_t push_ucs2_allocate(smb_ucs2_t **dest, const char *src)
489 size_t src_len = strlen(src)+1;
492 return convert_string_allocate(CH_UNIX, CH_UCS2, src, src_len, (void **)dest);
496 Copy a string from a char* src to a UTF-8 destination.
497 Return the number of bytes occupied by the string in the destination
499 STR_TERMINATE means include the null termination
500 STR_UPPER means uppercase in the destination
501 dest_len is the maximum length allowed in the destination. If dest_len
502 is -1 then no maxiumum is used.
505 static size_t push_utf8(void *dest, const char *src, size_t dest_len, int flags)
507 size_t src_len = strlen(src);
510 /* treat a pstring as "unlimited" length */
511 if (dest_len == (size_t)-1)
512 dest_len = sizeof(pstring);
514 if (flags & STR_UPPER) {
515 pstrcpy(tmpbuf, src);
520 if (flags & STR_TERMINATE)
523 return convert_string(CH_UNIX, CH_UTF8, src, src_len, dest, dest_len);
526 size_t push_utf8_fstring(void *dest, const char *src)
528 return push_utf8(dest, src, sizeof(fstring), STR_TERMINATE);
532 * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
534 * @param dest always set at least to NULL
536 * @returns The number of bytes occupied by the string in the destination
539 size_t push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
541 size_t src_len = strlen(src)+1;
544 return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len, (void**)dest);
548 * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer
550 * @param dest always set at least to NULL
552 * @returns The number of bytes occupied by the string in the destination
555 size_t push_utf8_allocate(char **dest, const char *src)
557 size_t src_len = strlen(src)+1;
560 return convert_string_allocate(CH_UNIX, CH_UTF8, src, src_len, (void **)dest);
564 Copy a string from a ucs2 source to a unix char* destination.
566 STR_TERMINATE means the string in src is null terminated.
567 STR_NOALIGN means don't try to align.
568 if STR_TERMINATE is set then src_len is ignored if it is -1.
569 src_len is the length of the source area in bytes
570 Return the number of bytes occupied by the string in src.
571 The resulting string in "dest" is always null terminated.
574 size_t pull_ucs2(const void *base_ptr, char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
578 if (dest_len == (size_t)-1)
579 dest_len = sizeof(pstring);
581 if (ucs2_align(base_ptr, src, flags)) {
582 src = (const void *)((const char *)src + 1);
587 if (flags & STR_TERMINATE) {
588 if (src_len == (size_t)-1) {
589 src_len = strlen_w(src)*2 + 2;
591 size_t len = strnlen_w(src, src_len/2);
598 /* ucs2 is always a multiple of 2 bytes */
599 if (src_len != (size_t)-1)
602 ret = convert_string(CH_UCS2, CH_UNIX, src, src_len, dest, dest_len);
604 dest[MIN(ret, dest_len-1)] = 0;
609 size_t pull_ucs2_pstring(char *dest, const void *src)
611 return pull_ucs2(NULL, dest, src, sizeof(pstring), -1, STR_TERMINATE);
614 size_t pull_ucs2_fstring(char *dest, const void *src)
616 return pull_ucs2(NULL, dest, src, sizeof(fstring), -1, STR_TERMINATE);
620 * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
622 * @param dest always set at least to NULL
624 * @returns The number of bytes occupied by the string in the destination
627 size_t pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src)
629 size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
631 return convert_string_talloc(ctx, CH_UCS2, CH_UNIX, src, src_len, (void **)dest);
635 * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer
637 * @param dest always set at least to NULL
639 * @returns The number of bytes occupied by the string in the destination
642 size_t pull_ucs2_allocate(void **dest, const smb_ucs2_t *src)
644 size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
646 return convert_string_allocate(CH_UCS2, CH_UNIX, src, src_len, dest);
650 * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
652 * @param dest always set at least to NULL
654 * @returns The number of bytes occupied by the string in the destination
657 size_t pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
659 size_t src_len = strlen(src)+1;
661 return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len, (void **)dest);
665 * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer
667 * @param dest always set at least to NULL
669 * @returns The number of bytes occupied by the string in the destination
672 size_t pull_utf8_allocate(void **dest, const char *src)
674 size_t src_len = strlen(src)+1;
676 return convert_string_allocate(CH_UTF8, CH_UNIX, src, src_len, dest);
680 Copy a string from a char* src to a unicode or ascii
681 dos codepage destination choosing unicode or ascii based on the
682 flags in the SMB buffer starting at base_ptr.
683 Return the number of bytes occupied by the string in the destination.
685 STR_TERMINATE means include the null termination.
686 STR_UPPER means uppercase in the destination.
687 STR_ASCII use ascii even with unicode packet.
688 STR_NOALIGN means don't do alignment.
689 dest_len is the maximum length allowed in the destination. If dest_len
690 is -1 then no maxiumum is used.
693 size_t push_string_fn(const char *function, unsigned int line, const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
695 if (dest_len != (size_t)-1)
696 clobber_region(function, line, dest, dest_len);
698 if (!(flags & STR_ASCII) && \
699 ((flags & STR_UNICODE || \
700 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
701 return push_ucs2(base_ptr, dest, src, dest_len, flags);
703 return push_ascii(dest, src, dest_len, flags);
708 Copy a string from a unicode or ascii source (depending on
709 the packet flags) to a char* destination.
711 STR_TERMINATE means the string in src is null terminated.
712 STR_UNICODE means to force as unicode.
713 STR_ASCII use ascii even with unicode packet.
714 STR_NOALIGN means don't do alignment.
715 if STR_TERMINATE is set then src_len is ignored is it is -1
716 src_len is the length of the source area in bytes.
717 Return the number of bytes occupied by the string in src.
718 The resulting string in "dest" is always null terminated.
721 size_t pull_string_fn(const char *function, unsigned int line, const void *base_ptr, char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
723 if (dest_len != (size_t)-1)
724 clobber_region(function, line, dest, dest_len);
726 if (!(flags & STR_ASCII) && \
727 ((flags & STR_UNICODE || \
728 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
729 return pull_ucs2(base_ptr, dest, src, dest_len, src_len, flags);
731 return pull_ascii(dest, src, dest_len, src_len, flags);
734 size_t align_string(const void *base_ptr, const char *p, int flags)
736 if (!(flags & STR_ASCII) && \
737 ((flags & STR_UNICODE || \
738 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
739 return ucs2_align(base_ptr, p, flags);
745 Convert from unix to ucs2 charset and return the
746 allocated and converted string or NULL if an error occurred.
747 You must provide a zero terminated string.
748 The returning string will be zero terminated.
751 smb_ucs2_t *acnv_uxu2(const char *src)
757 slen = strlen(src) + 1;
758 dlen = convert_string_allocate(CH_UNIX, CH_UCS2, src, slen, &dest);
759 if (dlen == (size_t)-1)
766 Convert from dos to ucs2 charset and return the
767 allocated and converted string or NULL if an error occurred.
768 You must provide a zero terminated string.
769 The returning string will be zero terminated.
772 smb_ucs2_t *acnv_dosu2(const char *src)
778 slen = strlen(src) + 1;
779 dlen = convert_string_allocate(CH_DOS, CH_UCS2, src, slen, &dest);
780 if (dlen == (size_t)-1)