source3/lib/iconv.c

   1 /*
   2    Unix SMB/CIFS implementation.
   3    minimal iconv implementation
   4    Copyright (C) Andrew Tridgell 2001
   5    Copyright (C) Jelmer Vernooij 2002,2003
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 2 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20 */
  21
  22 #include "includes.h"
  23
  24 /*
  25  * We have to use strcasecmp here as the character conversions
  26  * haven't been initialised yet. JRA.
  27  */
  28
  29 #undef strcasecmp
  30
  31 /**
  32  * @file
  33  *
  34  * @brief Samba wrapper/stub for iconv character set conversion.
  35  *
  36  * iconv is the XPG2 interface for converting between character
  37  * encodings.  This file provides a Samba wrapper around it, and also
  38  * a simple reimplementation that is used if the system does not
  39  * implement iconv.
  40  *
  41  * Samba only works with encodings that are supersets of ASCII: ascii
  42  * characters like whitespace can be tested for directly, multibyte
  43  * sequences start with a byte with the high bit set, and strings are
  44  * terminated by a nul byte.
  45  *
  46  * Note that the only function provided by iconv is conversion between
  47  * characters.  It doesn't directly support operations like
  48  * uppercasing or comparison.  We have to convert to UCS-2 and compare
  49  * there.
  50  *
  51  * @sa Samba Developers Guide
  52  **/
  53
  54 static size_t ascii_pull(void *,const char **, size_t *, char **, size_t *);
  55 static size_t ascii_push(void *,const char **, size_t *, char **, size_t *);
  56 static size_t latin1_push(void *,const char **, size_t *, char **, size_t *);
  57 static size_t  utf8_pull(void *,const char **, size_t *, char **, size_t *);
  58 static size_t  utf8_push(void *,const char **, size_t *, char **, size_t *);
  59 static size_t ucs2hex_pull(void *,const char **, size_t *, char **, size_t *);
  60 static size_t ucs2hex_push(void *,const char **, size_t *, char **, size_t *);
  61 static size_t iconv_copy(void *,const char **, size_t *, char **, size_t *);
  62 static size_t iconv_swab  (void *,const char **, size_t *, char **, size_t *);
  63
  64 static struct charset_functions builtin_functions[] = {
  65         /* windows is really neither UCS-2 not UTF-16 */
  66         {"UCS-2LE",  iconv_copy, iconv_copy},
  67         {"UTF-16LE",  iconv_copy, iconv_copy},
  68         {"UCS-2BE",  iconv_swab, iconv_swab},
  69
  70         /* we include the UTF-8 alias to cope with differing locale settings */
  71         {"UTF8",   utf8_pull,  utf8_push},
  72         {"UTF-8",   utf8_pull,  utf8_push},
  73         {"ASCII", ascii_pull, ascii_push},
  74         {"646", ascii_pull, ascii_push},
  75         {"ISO-8859-1", ascii_pull, latin1_push},
  76         {"UCS2-HEX", ucs2hex_pull, ucs2hex_push},
  77         {NULL, NULL, NULL}
  78 };
  79
  80 static struct charset_functions *charsets = NULL;
  81
  82 static struct charset_functions *find_charset_functions(const char *name)
  83 {
  84         struct charset_functions *c = charsets;
  85
  86         while(c) {
  87                 if (strcasecmp(name, c->name) == 0) {
  88                         return c;
  89                 }
  90                 c = c->next;
  91         }
  92
  93         return NULL;
  94 }
  95
  96 NTSTATUS smb_register_charset(struct charset_functions *funcs)
  97 {
  98         if (!funcs) {
  99                 return NT_STATUS_INVALID_PARAMETER;
 100         }
 101
 102         DEBUG(5, ("Attempting to register new charset %s\n", funcs->name));
 103         /* Check whether we already have this charset... */
 104         if (find_charset_functions(funcs->name)) {
 105                 DEBUG(0, ("Duplicate charset %s, not registering\n", funcs->name));
 106                 return NT_STATUS_OBJECT_NAME_COLLISION;
 107         }
 108
 109         funcs->next = funcs->prev = NULL;
 110         DEBUG(5, ("Registered charset %s\n", funcs->name));
 111         DLIST_ADD(charsets, funcs);
 112         return NT_STATUS_OK;
 113 }
 114
 115 static void lazy_initialize_iconv(void)
 116 {
 117         static BOOL initialized;
 118         int i;
 119
 120         if (!initialized) {
 121                 initialized = True;
 122                 for(i = 0; builtin_functions[i].name; i++)
 123                         smb_register_charset(&builtin_functions[i]);
 124                 static_init_charset;
 125         }
 126 }
 127
 128 /* if there was an error then reset the internal state,
 129    this ensures that we don't have a shift state remaining for
 130    character sets like SJIS */
 131 static size_t sys_iconv(void *cd,
 132                         const char **inbuf, size_t *inbytesleft,
 133                         char **outbuf, size_t *outbytesleft)
 134 {
 135 #ifdef HAVE_NATIVE_ICONV
 136         size_t ret = iconv((iconv_t)cd,
 137                            (char **)inbuf, inbytesleft,
 138                            outbuf, outbytesleft);
 139         if (ret == (size_t)-1) {
 140                 int saved_errno = errno;
 141                 iconv(cd, NULL, NULL, NULL, NULL);
 142                 errno = saved_errno;
 143         }
 144         return ret;
 145 #else
 146         errno = EINVAL;
 147         return -1;
 148 #endif
 149 }
 150
 151 /**
 152  * This is a simple portable iconv() implementaion.
 153  *
 154  * It only knows about a very small number of character sets - just
 155  * enough that Samba works on systems that don't have iconv.
 156  **/
 157 size_t smb_iconv(smb_iconv_t cd,
 158                  const char **inbuf, size_t *inbytesleft,
 159                  char **outbuf, size_t *outbytesleft)
 160 {
 161         char cvtbuf[2048];
 162         char *bufp = cvtbuf;
 163         size_t bufsize;
 164
 165         /* in many cases we can go direct */
 166         if (cd->direct) {
 167                 return cd->direct(cd->cd_direct,
 168                                   inbuf, inbytesleft, outbuf, outbytesleft);
 169         }
 170
 171
 172         /* otherwise we have to do it chunks at a time */
 173         while (*inbytesleft > 0) {
 174                 bufp = cvtbuf;
 175                 bufsize = sizeof(cvtbuf);
 176
 177                 if (cd->pull(cd->cd_pull,
 178                              inbuf, inbytesleft, &bufp, &bufsize) == -1
 179                     && errno != E2BIG) return -1;
 180
 181                 bufp = cvtbuf;
 182                 bufsize = sizeof(cvtbuf) - bufsize;
 183
 184                 if (cd->push(cd->cd_push,
 185                              (const char **)&bufp, &bufsize,
 186                              outbuf, outbytesleft) == -1) return -1;
 187         }
 188
 189         return 0;
 190 }
 191
 192 /*
 193   simple iconv_open() wrapper
 194  */
 195 smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
 196 {
 197         smb_iconv_t ret;
 198         struct charset_functions *from, *to;
 199
 200         lazy_initialize_iconv();
 201         from = charsets;
 202         to = charsets;
 203
 204         ret = (smb_iconv_t)malloc(sizeof(*ret));
 205         if (!ret) {
 206                 errno = ENOMEM;
 207                 return (smb_iconv_t)-1;
 208         }
 209         memset(ret, 0, sizeof(*ret));
 210
 211         ret->from_name = strdup(fromcode);
 212         ret->to_name = strdup(tocode);
 213
 214         /* check for the simplest null conversion */
 215         if (strcasecmp(fromcode, tocode) == 0) {
 216                 ret->direct = iconv_copy;
 217                 return ret;
 218         }
 219
 220         /* check if we have a builtin function for this conversion */
 221         from = find_charset_functions(fromcode);
 222         if(from)ret->pull = from->pull;
 223
 224         to = find_charset_functions(tocode);
 225         if(to)ret->push = to->push;
 226
 227         /* check if we can use iconv for this conversion */
 228 #ifdef HAVE_NATIVE_ICONV
 229         if (!ret->pull) {
 230                 ret->cd_pull = iconv_open("UCS-2LE", fromcode);
 231                 if (ret->cd_pull != (iconv_t)-1)
 232                         ret->pull = sys_iconv;
 233         }
 234
 235         if (!ret->push) {
 236                 ret->cd_push = iconv_open(tocode, "UCS-2LE");
 237                 if (ret->cd_push != (iconv_t)-1)
 238                         ret->push = sys_iconv;
 239         }
 240 #endif
 241
 242         /* check if there is a module available that can do this conversion */
 243         if (!ret->pull && NT_STATUS_IS_OK(smb_probe_module("charset", fromcode))) {
 244                 if(!(from = find_charset_functions(fromcode)))
 245                         DEBUG(0, ("Module %s doesn't provide charset %s!\n", fromcode, fromcode));
 246                 else
 247                         ret->pull = from->pull;
 248         }
 249
 250         if (!ret->push && NT_STATUS_IS_OK(smb_probe_module("charset", tocode))) {
 251                 if(!(to = find_charset_functions(tocode)))
 252                         DEBUG(0, ("Module %s doesn't provide charset %s!\n", tocode, tocode));
 253                 else
 254                         ret->push = to->push;
 255         }
 256
 257         if (!ret->push || !ret->pull) {
 258                 SAFE_FREE(ret->from_name);
 259                 SAFE_FREE(ret->to_name);
 260                 SAFE_FREE(ret);
 261                 errno = EINVAL;
 262                 return (smb_iconv_t)-1;
 263         }
 264
 265         /* check for conversion to/from ucs2 */
 266         if (strcasecmp(fromcode, "UCS-2LE") == 0 && to) {
 267                 ret->direct = to->push;
 268                 ret->push = ret->pull = NULL;
 269                 return ret;
 270         }
 271
 272         if (strcasecmp(tocode, "UCS-2LE") == 0 && from) {
 273                 ret->direct = from->pull;
 274                 ret->push = ret->pull = NULL;
 275                 return ret;
 276         }
 277
 278         /* Check if we can do the conversion direct */
 279 #ifdef HAVE_NATIVE_ICONV
 280         if (strcasecmp(fromcode, "UCS-2LE") == 0) {
 281                 ret->direct = sys_iconv;
 282                 ret->cd_direct = ret->cd_push;
 283                 ret->cd_push = NULL;
 284                 return ret;
 285         }
 286         if (strcasecmp(tocode, "UCS-2LE") == 0) {
 287                 ret->direct = sys_iconv;
 288                 ret->cd_direct = ret->cd_pull;
 289                 ret->cd_pull = NULL;
 290                 return ret;
 291         }
 292 #endif
 293
 294         return ret;
 295 }
 296
 297 /*
 298   simple iconv_close() wrapper
 299 */
 300 int smb_iconv_close (smb_iconv_t cd)
 301 {
 302 #ifdef HAVE_NATIVE_ICONV
 303         if (cd->cd_direct) iconv_close((iconv_t)cd->cd_direct);
 304         if (cd->cd_pull) iconv_close((iconv_t)cd->cd_pull);
 305         if (cd->cd_push) iconv_close((iconv_t)cd->cd_push);
 306 #endif
 307
 308         SAFE_FREE(cd->from_name);
 309         SAFE_FREE(cd->to_name);
 310
 311         memset(cd, 0, sizeof(*cd));
 312         SAFE_FREE(cd);
 313         return 0;
 314 }
 315
 316
 317 /**********************************************************************
 318  the following functions implement the builtin character sets in Samba
 319  and also the "test" character sets that are designed to test
 320  multi-byte character set support for english users
 321 ***********************************************************************/
 322
 323 static size_t ascii_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 324                          char **outbuf, size_t *outbytesleft)
 325 {
 326         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 327                 (*outbuf)[0] = (*inbuf)[0];
 328                 (*outbuf)[1] = 0;
 329                 (*inbytesleft)  -= 1;
 330                 (*outbytesleft) -= 2;
 331                 (*inbuf)  += 1;
 332                 (*outbuf) += 2;
 333         }
 334
 335         if (*inbytesleft > 0) {
 336                 errno = E2BIG;
 337                 return -1;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static size_t ascii_push(void *cd, const char **inbuf, size_t *inbytesleft,
 344                          char **outbuf, size_t *outbytesleft)
 345 {
 346         int ir_count=0;
 347
 348         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 349                 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
 350                 if ((*inbuf)[1]) ir_count++;
 351                 (*inbytesleft)  -= 2;
 352                 (*outbytesleft) -= 1;
 353                 (*inbuf)  += 2;
 354                 (*outbuf) += 1;
 355         }
 356
 357         if (*inbytesleft == 1) {
 358                 errno = EINVAL;
 359                 return -1;
 360         }
 361
 362         if (*inbytesleft > 1) {
 363                 errno = E2BIG;
 364                 return -1;
 365         }
 366
 367         return ir_count;
 368 }
 369
 370 static size_t latin1_push(void *cd, const char **inbuf, size_t *inbytesleft,
 371                          char **outbuf, size_t *outbytesleft)
 372 {
 373         int ir_count=0;
 374
 375         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 376                 (*outbuf)[0] = (*inbuf)[0];
 377                 if ((*inbuf)[1]) ir_count++;
 378                 (*inbytesleft)  -= 2;
 379                 (*outbytesleft) -= 1;
 380                 (*inbuf)  += 2;
 381                 (*outbuf) += 1;
 382         }
 383
 384         if (*inbytesleft == 1) {
 385                 errno = EINVAL;
 386                 return -1;
 387         }
 388
 389         if (*inbytesleft > 1) {
 390                 errno = E2BIG;
 391                 return -1;
 392         }
 393
 394         return ir_count;
 395 }
 396
 397 static size_t ucs2hex_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 398                          char **outbuf, size_t *outbytesleft)
 399 {
 400         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 401                 unsigned v;
 402
 403                 if ((*inbuf)[0] != '@') {
 404                         /* seven bit ascii case */
 405                         (*outbuf)[0] = (*inbuf)[0];
 406                         (*outbuf)[1] = 0;
 407                         (*inbytesleft)  -= 1;
 408                         (*outbytesleft) -= 2;
 409                         (*inbuf)  += 1;
 410                         (*outbuf) += 2;
 411                         continue;
 412                 }
 413                 /* it's a hex character */
 414                 if (*inbytesleft < 5) {
 415                         errno = EINVAL;
 416                         return -1;
 417                 }
 418
 419                 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
 420                         errno = EILSEQ;
 421                         return -1;
 422                 }
 423
 424                 (*outbuf)[0] = v&0xff;
 425                 (*outbuf)[1] = v>>8;
 426                 (*inbytesleft)  -= 5;
 427                 (*outbytesleft) -= 2;
 428                 (*inbuf)  += 5;
 429                 (*outbuf) += 2;
 430         }
 431
 432         if (*inbytesleft > 0) {
 433                 errno = E2BIG;
 434                 return -1;
 435         }
 436
 437         return 0;
 438 }
 439
 440 static size_t ucs2hex_push(void *cd, const char **inbuf, size_t *inbytesleft,
 441                            char **outbuf, size_t *outbytesleft)
 442 {
 443         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 444                 char buf[6];
 445
 446                 if ((*inbuf)[1] == 0 &&
 447                     ((*inbuf)[0] & 0x80) == 0 &&
 448                     (*inbuf)[0] != '@') {
 449                         (*outbuf)[0] = (*inbuf)[0];
 450                         (*inbytesleft)  -= 2;
 451                         (*outbytesleft) -= 1;
 452                         (*inbuf)  += 2;
 453                         (*outbuf) += 1;
 454                         continue;
 455                 }
 456                 if (*outbytesleft < 5) {
 457                         errno = E2BIG;
 458                         return -1;
 459                 }
 460                 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
 461                 memcpy(*outbuf, buf, 5);
 462                 (*inbytesleft)  -= 2;
 463                 (*outbytesleft) -= 5;
 464                 (*inbuf)  += 2;
 465                 (*outbuf) += 5;
 466         }
 467
 468         if (*inbytesleft == 1) {
 469                 errno = EINVAL;
 470                 return -1;
 471         }
 472
 473         if (*inbytesleft > 1) {
 474                 errno = E2BIG;
 475                 return -1;
 476         }
 477
 478         return 0;
 479 }
 480
 481 static size_t iconv_swab(void *cd, const char **inbuf, size_t *inbytesleft,
 482                          char **outbuf, size_t *outbytesleft)
 483 {
 484         int n;
 485
 486         n = MIN(*inbytesleft, *outbytesleft);
 487
 488         swab(*inbuf, *outbuf, (n&~1));
 489         if (n&1) {
 490                 (*outbuf)[n-1] = 0;
 491         }
 492
 493         (*inbytesleft) -= n;
 494         (*outbytesleft) -= n;
 495         (*inbuf) += n;
 496         (*outbuf) += n;
 497
 498         if (*inbytesleft > 0) {
 499                 errno = E2BIG;
 500                 return -1;
 501         }
 502
 503         return 0;
 504 }
 505
 506 static size_t iconv_copy(void *cd, const char **inbuf, size_t *inbytesleft,
 507                          char **outbuf, size_t *outbytesleft)
 508 {
 509         int n;
 510
 511         n = MIN(*inbytesleft, *outbytesleft);
 512
 513         memmove(*outbuf, *inbuf, n);
 514
 515         (*inbytesleft) -= n;
 516         (*outbytesleft) -= n;
 517         (*inbuf) += n;
 518         (*outbuf) += n;
 519
 520         if (*inbytesleft > 0) {
 521                 errno = E2BIG;
 522                 return -1;
 523         }
 524
 525         return 0;
 526 }
 527
 528 static size_t utf8_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 529                          char **outbuf, size_t *outbytesleft)
 530 {
 531         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 532                 const unsigned char *c = (const unsigned char *)*inbuf;
 533                 unsigned char *uc = (unsigned char *)*outbuf;
 534                 int len = 1;
 535
 536                 if ((c[0] & 0x80) == 0) {
 537                         uc[0] = c[0];
 538                         uc[1] = 0;
 539                 } else if ((c[0] & 0xf0) == 0xe0) {
 540                         if (*inbytesleft < 3) {
 541                                 DEBUG(0,("short utf8 char\n"));
 542                                 goto badseq;
 543                         }
 544                         uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
 545                         uc[0] = (c[1]<<6) | (c[2]&0x3f);
 546                         len = 3;
 547                 } else if ((c[0] & 0xe0) == 0xc0) {
 548                         if (*inbytesleft < 2) {
 549                                 DEBUG(0,("short utf8 char\n"));
 550                                 goto badseq;
 551                         }
 552                         uc[1] = (c[0]>>2) & 0x7;
 553                         uc[0] = (c[0]<<6) | (c[1]&0x3f);
 554                         len = 2;
 555                 }
 556
 557                 (*inbuf)  += len;
 558                 (*inbytesleft)  -= len;
 559                 (*outbytesleft) -= 2;
 560                 (*outbuf) += 2;
 561         }
 562
 563         if (*inbytesleft > 0) {
 564                 errno = E2BIG;
 565                 return -1;
 566         }
 567
 568         return 0;
 569
 570 badseq:
 571         errno = EINVAL;
 572         return -1;
 573 }
 574
 575 static size_t utf8_push(void *cd, const char **inbuf, size_t *inbytesleft,
 576                          char **outbuf, size_t *outbytesleft)
 577 {
 578         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 579                 unsigned char *c = (unsigned char *)*outbuf;
 580                 const unsigned char *uc = (const unsigned char *)*inbuf;
 581                 int len=1;
 582
 583                 if (uc[1] & 0xf8) {
 584                         if (*outbytesleft < 3) {
 585                                 DEBUG(0,("short utf8 write\n"));
 586                                 goto toobig;
 587                         }
 588                         c[0] = 0xe0 | (uc[1]>>4);
 589                         c[1] = 0x80 | ((uc[1]&0xF)<<2) | (uc[0]>>6);
 590                         c[2] = 0x80 | (uc[0]&0x3f);
 591                         len = 3;
 592                 } else if (uc[1] | (uc[0] & 0x80)) {
 593                         if (*outbytesleft < 2) {
 594                                 DEBUG(0,("short utf8 write\n"));
 595                                 goto toobig;
 596                         }
 597                         c[0] = 0xc0 | (uc[1]<<2) | (uc[0]>>6);
 598                         c[1] = 0x80 | (uc[0]&0x3f);
 599                         len = 2;
 600                 } else {
 601                         c[0] = uc[0];
 602                 }
 603
 604
 605                 (*inbytesleft)  -= 2;
 606                 (*outbytesleft) -= len;
 607                 (*inbuf)  += 2;
 608                 (*outbuf) += len;
 609         }
 610
 611         if (*inbytesleft == 1) {
 612                 errno = EINVAL;
 613                 return -1;
 614         }
 615
 616         if (*inbytesleft > 1) {
 617                 errno = E2BIG;
 618                 return -1;
 619         }
 620
 621         return 0;
 622
 623 toobig:
 624         errno = E2BIG;
 625         return -1;
 626 }