2 Unix SMB/CIFS implementation.
3 Character set conversion Extensions
4 Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
5 Copyright (C) Andrew Tridgell 2001
6 Copyright (C) Simo Sorce 2001
8 This program is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 2 of the License, or
11 (at your option) any later version.
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 * @brief Character-set conversion routines built on our iconv.
30 * @note Samba's internal character set (at least in the 3.0 series)
31 * is always the same as the one for the Unix filesystem. It is
32 * <b>not</b> necessarily UTF-8 and may be different on machines that
33 * need i18n filenames to be compatible with Unix software. It does
34 * have to be a superset of ASCII. All multibyte sequences must start
35 * with a byte with the high bit set.
40 static pstring cvtbuf;
42 static smb_iconv_t conv_handles[NUM_CHARSETS][NUM_CHARSETS];
46 * Return the name of a charset to give to iconv().
48 static const char *charset_name(charset_t ch)
50 const char *ret = NULL;
52 if (ch == CH_UCS2) ret = "UCS-2LE";
53 else if (ch == CH_UNIX) ret = lp_unix_charset();
54 else if (ch == CH_DOS) ret = lp_dos_charset();
55 else if (ch == CH_DISPLAY) ret = lp_display_charset();
56 else if (ch == CH_UTF8) ret = "UTF8";
58 if (!ret || !*ret) ret = "ASCII";
62 void lazy_initialize_conv(void)
64 static int initialized = False;
74 * Initialize iconv conversion descriptors.
76 * This is called the first time it is needed, and also called again
77 * every time the configuration is reloaded, because the charset or
78 * codepage might have changed.
83 BOOL did_reload = False;
85 /* so that charset_name() works we need to get the UNIX<->UCS2 going
87 if (!conv_handles[CH_UNIX][CH_UCS2])
88 conv_handles[CH_UNIX][CH_UCS2] = smb_iconv_open("UCS-2LE", "ASCII");
90 if (!conv_handles[CH_UCS2][CH_UNIX])
91 conv_handles[CH_UCS2][CH_UNIX] = smb_iconv_open("ASCII", "UCS-2LE");
93 for (c1=0;c1<NUM_CHARSETS;c1++) {
94 for (c2=0;c2<NUM_CHARSETS;c2++) {
95 const char *n1 = charset_name((charset_t)c1);
96 const char *n2 = charset_name((charset_t)c2);
97 if (conv_handles[c1][c2] &&
98 strcmp(n1, conv_handles[c1][c2]->from_name) == 0 &&
99 strcmp(n2, conv_handles[c1][c2]->to_name) == 0)
104 if (conv_handles[c1][c2])
105 smb_iconv_close(conv_handles[c1][c2]);
107 conv_handles[c1][c2] = smb_iconv_open(n2,n1);
108 if (conv_handles[c1][c2] == (smb_iconv_t)-1) {
109 DEBUG(0,("Conversion from %s to %s not supported\n",
110 charset_name((charset_t)c1), charset_name((charset_t)c2)));
111 conv_handles[c1][c2] = NULL;
117 /* XXX: Does this really get called every time the dos
118 * codepage changes? */
119 /* XXX: Is the did_reload test too strict? */
120 init_doschar_table();
126 * Convert string from one encoding to another, making error checking etc
128 * @param src pointer to source string (multibyte or singlebyte)
129 * @param srclen length of the source string in bytes
130 * @param dest pointer to destination string (multibyte or singlebyte)
131 * @param destlen maximal length allowed for string
132 * @returns the number of bytes occupied in the destination
134 size_t convert_string(charset_t from, charset_t to,
135 void const *src, size_t srclen,
136 void *dest, size_t destlen)
140 const char* inbuf = (const char*)src;
141 char* outbuf = (char*)dest;
142 smb_iconv_t descriptor;
144 if (srclen == (size_t)-1)
145 srclen = strlen(src)+1;
147 lazy_initialize_conv();
149 descriptor = conv_handles[from][to];
151 if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
152 /* conversion not supported, use as is */
153 size_t len = MIN(srclen,destlen);
154 memcpy(dest,src,len);
160 retval = smb_iconv(descriptor, &inbuf, &i_len, &outbuf, &o_len);
161 if(retval==(size_t)-1) {
162 const char *reason="unknown error";
165 reason="Incomplete multibyte sequence";
168 reason="No more room";
169 DEBUG(0, ("convert_string: Required %d, available %d\n",
171 /* we are not sure we need srclen bytes,
172 may be more, may be less.
173 We only know we need more than destlen
177 reason="Illegal multibyte sequence";
180 /* smb_panic(reason); */
182 return destlen-o_len;
186 * Convert between character sets, allocating a new buffer for the result.
188 * @param srclen length of source buffer.
189 * @param dest always set at least to NULL
190 * @note -1 is not accepted for srclen.
192 * @returns Size in bytes of the converted string; or -1 in case of error.
195 size_t convert_string_allocate(charset_t from, charset_t to,
196 void const *src, size_t srclen, void **dest)
198 size_t i_len, o_len, destlen;
200 const char *inbuf = (const char *)src;
202 smb_iconv_t descriptor;
206 if (src == NULL || srclen == (size_t)-1)
209 lazy_initialize_conv();
211 descriptor = conv_handles[from][to];
213 if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
214 /* conversion not supported, return -1*/
215 DEBUG(3, ("convert_string_allocate: conversion not supported!\n"));
219 destlen = MAX(srclen, 512);
222 destlen = destlen * 2;
223 ob = (char *)Realloc(outbuf, destlen);
225 DEBUG(0, ("convert_string_allocate: realloc failed!\n"));
233 retval = smb_iconv(descriptor,
236 if(retval == (size_t)-1) {
237 const char *reason="unknown error";
240 reason="Incomplete multibyte sequence";
245 reason="Illegal multibyte sequence";
248 DEBUG(0,("Conversion error: %s(%s)\n",reason,inbuf));
249 /* smb_panic(reason); */
253 destlen = destlen - o_len;
254 *dest = (char *)Realloc(ob,destlen);
256 DEBUG(0, ("convert_string_allocate: out of memory!\n"));
266 * Convert between character sets, allocating a new buffer using talloc for the result.
268 * @param srclen length of source buffer.
269 * @param dest always set at least to NULL
270 * @note -1 is not accepted for srclen.
272 * @returns Size in bytes of the converted string; or -1 in case of error.
274 static size_t convert_string_talloc(TALLOC_CTX *ctx, charset_t from, charset_t to,
275 void const *src, size_t srclen, void **dest)
277 void *alloced_string;
281 dest_len=convert_string_allocate(from, to, src, srclen, &alloced_string);
282 if (dest_len == (size_t)-1)
284 *dest = talloc_memdup(ctx, alloced_string, dest_len);
285 SAFE_FREE(alloced_string);
291 size_t unix_strupper(const char *src, size_t srclen, char *dest, size_t destlen)
294 smb_ucs2_t *buffer=(smb_ucs2_t*)cvtbuf;
295 size=convert_string(CH_UNIX, CH_UCS2, src, srclen, buffer, sizeof(cvtbuf));
296 if (!strupper_w(buffer) && (dest == src))
298 return convert_string(CH_UCS2, CH_UNIX, buffer, size, dest, destlen);
301 size_t unix_strlower(const char *src, size_t srclen, char *dest, size_t destlen)
304 smb_ucs2_t *buffer=(smb_ucs2_t*)cvtbuf;
305 size=convert_string(CH_UNIX, CH_UCS2, src, srclen, buffer, sizeof(cvtbuf));
306 if (!strlower_w(buffer) && (dest == src))
308 return convert_string(CH_UCS2, CH_UNIX, buffer, size, dest, destlen);
312 static size_t ucs2_align(const void *base_ptr, const void *p, int flags)
314 if (flags & (STR_NOALIGN|STR_ASCII))
316 return PTR_DIFF(p, base_ptr) & 1;
321 * Copy a string from a char* unix src to a dos codepage string destination.
323 * @return the number of bytes occupied by the string in the destination.
325 * @param flags can include
327 * <dt>STR_TERMINATE</dt> <dd>means include the null termination</dd>
328 * <dt>STR_UPPER</dt> <dd>means uppercase in the destination</dd>
331 * @param dest_len the maximum length in bytes allowed in the
332 * destination. If @p dest_len is -1 then no maximum is used.
334 size_t push_ascii(void *dest, const char *src, size_t dest_len, int flags)
336 size_t src_len = strlen(src);
339 /* treat a pstring as "unlimited" length */
340 if (dest_len == (size_t)-1)
341 dest_len = sizeof(pstring);
343 if (flags & STR_UPPER) {
344 pstrcpy(tmpbuf, src);
349 if (flags & STR_TERMINATE)
352 return convert_string(CH_UNIX, CH_DOS, src, src_len, dest, dest_len);
355 size_t push_ascii_fstring(void *dest, const char *src)
357 return push_ascii(dest, src, sizeof(fstring), STR_TERMINATE);
360 size_t push_ascii_pstring(void *dest, const char *src)
362 return push_ascii(dest, src, sizeof(pstring), STR_TERMINATE);
366 * Copy a string from a dos codepage source to a unix char* destination.
368 * The resulting string in "dest" is always null terminated.
370 * @param flags can have:
372 * <dt>STR_TERMINATE</dt>
373 * <dd>STR_TERMINATE means the string in @p src
374 * is null terminated, and src_len is ignored.</dd>
377 * @param src_len is the length of the source area in bytes.
378 * @returns the number of bytes occupied by the string in @p src.
380 size_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
384 if (dest_len == (size_t)-1)
385 dest_len = sizeof(pstring);
387 if (flags & STR_TERMINATE) {
388 if (src_len == (size_t)-1) {
389 src_len = strlen(src) + 1;
391 size_t len = strnlen(src, src_len);
398 ret = convert_string(CH_DOS, CH_UNIX, src, src_len, dest, dest_len);
401 dest[MIN(ret, dest_len-1)] = 0;
406 size_t pull_ascii_pstring(char *dest, const void *src)
408 return pull_ascii(dest, src, sizeof(pstring), -1, STR_TERMINATE);
411 size_t pull_ascii_fstring(char *dest, const void *src)
413 return pull_ascii(dest, src, sizeof(fstring), -1, STR_TERMINATE);
417 * Copy a string from a char* src to a unicode destination.
419 * @returns the number of bytes occupied by the string in the destination.
421 * @param flags can have:
424 * <dt>STR_TERMINATE <dd>means include the null termination.
425 * <dt>STR_UPPER <dd>means uppercase in the destination.
426 * <dt>STR_NOALIGN <dd>means don't do alignment.
429 * @param dest_len is the maximum length allowed in the
430 * destination. If dest_len is -1 then no maxiumum is used.
432 size_t push_ucs2(const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
435 size_t src_len = strlen(src);
438 /* treat a pstring as "unlimited" length */
439 if (dest_len == (size_t)-1)
440 dest_len = sizeof(pstring);
442 if (flags & STR_UPPER) {
443 pstrcpy(tmpbuf, src);
448 if (flags & STR_TERMINATE)
451 if (ucs2_align(base_ptr, dest, flags)) {
453 dest = (void *)((char *)dest + 1);
454 if (dest_len) dest_len--;
458 /* ucs2 is always a multiple of 2 bytes */
461 len += convert_string(CH_UNIX, CH_UCS2, src, src_len, dest, dest_len);
467 * Copy a string from a unix char* src to a UCS2 destination,
468 * allocating a buffer using talloc().
470 * @param dest always set at least to NULL
472 * @returns The number of bytes occupied by the string in the destination
473 * or -1 in case of error.
475 size_t push_ucs2_talloc(TALLOC_CTX *ctx, smb_ucs2_t **dest, const char *src)
477 size_t src_len = strlen(src)+1;
480 return convert_string_talloc(ctx, CH_UNIX, CH_UCS2, src, src_len, (void **)dest);
485 * Copy a string from a unix char* src to a UCS2 destination, allocating a buffer
487 * @param dest always set at least to NULL
489 * @returns The number of bytes occupied by the string in the destination
490 * or -1 in case of error.
493 size_t push_ucs2_allocate(smb_ucs2_t **dest, const char *src)
495 size_t src_len = strlen(src)+1;
498 return convert_string_allocate(CH_UNIX, CH_UCS2, src, src_len, (void **)dest);
502 Copy a string from a char* src to a UTF-8 destination.
503 Return the number of bytes occupied by the string in the destination
505 STR_TERMINATE means include the null termination
506 STR_UPPER means uppercase in the destination
507 dest_len is the maximum length allowed in the destination. If dest_len
508 is -1 then no maxiumum is used.
511 static size_t push_utf8(void *dest, const char *src, size_t dest_len, int flags)
513 size_t src_len = strlen(src);
516 /* treat a pstring as "unlimited" length */
517 if (dest_len == (size_t)-1)
518 dest_len = sizeof(pstring);
520 if (flags & STR_UPPER) {
521 pstrcpy(tmpbuf, src);
526 if (flags & STR_TERMINATE)
529 return convert_string(CH_UNIX, CH_UTF8, src, src_len, dest, dest_len);
532 size_t push_utf8_fstring(void *dest, const char *src)
534 return push_utf8(dest, src, sizeof(fstring), STR_TERMINATE);
538 * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
540 * @param dest always set at least to NULL
542 * @returns The number of bytes occupied by the string in the destination
545 size_t push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
547 size_t src_len = strlen(src)+1;
550 return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len, (void**)dest);
554 * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer
556 * @param dest always set at least to NULL
558 * @returns The number of bytes occupied by the string in the destination
561 size_t push_utf8_allocate(char **dest, const char *src)
563 size_t src_len = strlen(src)+1;
566 return convert_string_allocate(CH_UNIX, CH_UTF8, src, src_len, (void **)dest);
570 Copy a string from a ucs2 source to a unix char* destination.
572 STR_TERMINATE means the string in src is null terminated.
573 STR_NOALIGN means don't try to align.
574 if STR_TERMINATE is set then src_len is ignored if it is -1.
575 src_len is the length of the source area in bytes
576 Return the number of bytes occupied by the string in src.
577 The resulting string in "dest" is always null terminated.
580 size_t pull_ucs2(const void *base_ptr, char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
584 if (dest_len == (size_t)-1)
585 dest_len = sizeof(pstring);
587 if (ucs2_align(base_ptr, src, flags)) {
588 src = (const void *)((const char *)src + 1);
593 if (flags & STR_TERMINATE) {
594 if (src_len == (size_t)-1) {
595 src_len = strlen_w(src)*2 + 2;
597 size_t len = strnlen_w(src, src_len/2);
604 /* ucs2 is always a multiple of 2 bytes */
605 if (src_len != (size_t)-1)
608 ret = convert_string(CH_UCS2, CH_UNIX, src, src_len, dest, dest_len);
610 dest[MIN(ret, dest_len-1)] = 0;
615 size_t pull_ucs2_pstring(char *dest, const void *src)
617 return pull_ucs2(NULL, dest, src, sizeof(pstring), -1, STR_TERMINATE);
620 size_t pull_ucs2_fstring(char *dest, const void *src)
622 return pull_ucs2(NULL, dest, src, sizeof(fstring), -1, STR_TERMINATE);
626 * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
628 * @param dest always set at least to NULL
630 * @returns The number of bytes occupied by the string in the destination
633 size_t pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const smb_ucs2_t *src)
635 size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
637 return convert_string_talloc(ctx, CH_UCS2, CH_UNIX, src, src_len, (void **)dest);
641 * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer
643 * @param dest always set at least to NULL
645 * @returns The number of bytes occupied by the string in the destination
648 size_t pull_ucs2_allocate(char **dest, const smb_ucs2_t *src)
650 size_t src_len = (strlen_w(src)+1) * sizeof(smb_ucs2_t);
652 return convert_string_allocate(CH_UCS2, CH_UNIX, src, src_len, (void **)dest);
656 * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
658 * @param dest always set at least to NULL
660 * @returns The number of bytes occupied by the string in the destination
663 size_t pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
665 size_t src_len = strlen(src)+1;
667 return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len, (void **)dest);
671 * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer
673 * @param dest always set at least to NULL
675 * @returns The number of bytes occupied by the string in the destination
678 size_t pull_utf8_allocate(void **dest, const char *src)
680 size_t src_len = strlen(src)+1;
682 return convert_string_allocate(CH_UTF8, CH_UNIX, src, src_len, dest);
686 Copy a string from a char* src to a unicode or ascii
687 dos codepage destination choosing unicode or ascii based on the
688 flags in the SMB buffer starting at base_ptr.
689 Return the number of bytes occupied by the string in the destination.
691 STR_TERMINATE means include the null termination.
692 STR_UPPER means uppercase in the destination.
693 STR_ASCII use ascii even with unicode packet.
694 STR_NOALIGN means don't do alignment.
695 dest_len is the maximum length allowed in the destination. If dest_len
696 is -1 then no maxiumum is used.
699 size_t push_string_fn(const char *function, unsigned int line, const void *base_ptr, void *dest, const char *src, size_t dest_len, int flags)
701 if (dest_len != (size_t)-1)
702 clobber_region(function, line, dest, dest_len);
704 if (!(flags & STR_ASCII) && \
705 ((flags & STR_UNICODE || \
706 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
707 return push_ucs2(base_ptr, dest, src, dest_len, flags);
709 return push_ascii(dest, src, dest_len, flags);
714 Copy a string from a unicode or ascii source (depending on
715 the packet flags) to a char* destination.
717 STR_TERMINATE means the string in src is null terminated.
718 STR_UNICODE means to force as unicode.
719 STR_ASCII use ascii even with unicode packet.
720 STR_NOALIGN means don't do alignment.
721 if STR_TERMINATE is set then src_len is ignored is it is -1
722 src_len is the length of the source area in bytes.
723 Return the number of bytes occupied by the string in src.
724 The resulting string in "dest" is always null terminated.
727 size_t pull_string_fn(const char *function, unsigned int line, const void *base_ptr, char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
729 if (dest_len != (size_t)-1)
730 clobber_region(function, line, dest, dest_len);
732 if (!(flags & STR_ASCII) && \
733 ((flags & STR_UNICODE || \
734 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
735 return pull_ucs2(base_ptr, dest, src, dest_len, src_len, flags);
737 return pull_ascii(dest, src, dest_len, src_len, flags);
740 size_t align_string(const void *base_ptr, const char *p, int flags)
742 if (!(flags & STR_ASCII) && \
743 ((flags & STR_UNICODE || \
744 (SVAL(base_ptr, smb_flg2) & FLAGS2_UNICODE_STRINGS)))) {
745 return ucs2_align(base_ptr, p, flags);
751 Convert from unix to ucs2 charset and return the
752 allocated and converted string or NULL if an error occurred.
753 You must provide a zero terminated string.
754 The returning string will be zero terminated.
757 smb_ucs2_t *acnv_uxu2(const char *src)
763 slen = strlen(src) + 1;
764 dlen = convert_string_allocate(CH_UNIX, CH_UCS2, src, slen, &dest);
765 if (dlen == (size_t)-1)
772 Convert from dos to ucs2 charset and return the
773 allocated and converted string or NULL if an error occurred.
774 You must provide a zero terminated string.
775 The returning string will be zero terminated.
778 smb_ucs2_t *acnv_dosu2(const char *src)
784 slen = strlen(src) + 1;
785 dlen = convert_string_allocate(CH_DOS, CH_UCS2, src, slen, &dest);
786 if (dlen == (size_t)-1)