source4/lib/iconv.c

   1 /*
   2    Unix SMB/CIFS implementation.
   3    minimal iconv implementation
   4    Copyright (C) Andrew Tridgell 2001
   5    Copyright (C) Jelmer Vernooij 2002
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 2 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20 */
  21
  22 #include "includes.h"
  23
  24
  25 /**
  26  * @file
  27  *
  28  * @brief Samba wrapper/stub for iconv character set conversion.
  29  *
  30  * iconv is the XPG2 interface for converting between character
  31  * encodings.  This file provides a Samba wrapper around it, and also
  32  * a simple reimplementation that is used if the system does not
  33  * implement iconv.
  34  *
  35  * Samba only works with encodings that are supersets of ASCII: ascii
  36  * characters like whitespace can be tested for directly, multibyte
  37  * sequences start with a byte with the high bit set, and strings are
  38  * terminated by a nul byte.
  39  *
  40  * Note that the only function provided by iconv is conversion between
  41  * characters.  It doesn't directly support operations like
  42  * uppercasing or comparison.  We have to convert to UTF-16LE and
  43  * compare there.
  44  *
  45  * @sa Samba Developers Guide
  46  **/
  47
  48 static size_t ascii_pull  (void *,const char **, size_t *, char **, size_t *);
  49 static size_t ascii_push  (void *,const char **, size_t *, char **, size_t *);
  50 static size_t utf8_pull   (void *,const char **, size_t *, char **, size_t *);
  51 static size_t utf8_push   (void *,const char **, size_t *, char **, size_t *);
  52 static size_t ucs2hex_pull(void *,const char **, size_t *, char **, size_t *);
  53 static size_t ucs2hex_push(void *,const char **, size_t *, char **, size_t *);
  54 static size_t iconv_copy  (void *,const char **, size_t *, char **, size_t *);
  55 static size_t iconv_swab  (void *,const char **, size_t *, char **, size_t *);
  56
  57 static const struct charset_functions const builtin_functions[] = {
  58         /* windows is closest to UTF-16 */
  59         {"UCS-2LE",  iconv_copy, iconv_copy},
  60         {"UTF-16LE",  iconv_copy, iconv_copy},
  61         {"UCS-2BE",  iconv_swab, iconv_swab},
  62         {"UTF-16BE",  iconv_swab, iconv_swab},
  63
  64         /* we include the UTF-8 alias to cope with differing locale settings */
  65         {"UTF8",   utf8_pull,  utf8_push},
  66         {"UTF-8",   utf8_pull,  utf8_push},
  67         {"ASCII", ascii_pull, ascii_push},
  68         {"UCS2-HEX", ucs2hex_pull, ucs2hex_push},
  69         {NULL, NULL, NULL}
  70 };
  71
  72 static struct charset_functions *charsets = NULL;
  73
  74 static NTSTATUS charset_register_backend(const void *_funcs)
  75 {
  76         struct charset_functions *funcs = memdup(_funcs,sizeof(struct charset_functions));
  77         struct charset_functions *c = charsets;
  78
  79         /* Check whether we already have this charset... */
  80         while(c) {
  81                 if(!strcasecmp(c->name, funcs->name)){
  82                         DEBUG(2, ("Duplicate charset %s, not registering\n", funcs->name));
  83                         return NT_STATUS_OBJECT_NAME_COLLISION;
  84                 }
  85                 c = c->next;
  86         }
  87
  88         funcs->next = funcs->prev = NULL;
  89         DLIST_ADD(charsets, funcs);
  90         return NT_STATUS_OK;
  91 }
  92
  93 static void lazy_initialize_iconv(void)
  94 {
  95         static BOOL initialized = False;
  96         int i;
  97
  98         if (!initialized) {
  99                 initialized = True;
 100                 register_subsystem("charset", charset_register_backend);
 101
 102                 for(i = 0; builtin_functions[i].name; i++)
 103                         register_backend("charset", &builtin_functions[i]);
 104         }
 105 }
 106
 107 #ifdef HAVE_NATIVE_ICONV
 108 /* if there was an error then reset the internal state,
 109    this ensures that we don't have a shift state remaining for
 110    character sets like SJIS */
 111 static size_t sys_iconv(void *cd,
 112                         const char **inbuf, size_t *inbytesleft,
 113                         char **outbuf, size_t *outbytesleft)
 114 {
 115         size_t ret = iconv((iconv_t)cd,
 116                            discard_const_p(char *, inbuf), inbytesleft,
 117                            outbuf, outbytesleft);
 118         if (ret == (size_t)-1) iconv(cd, NULL, NULL, NULL, NULL);
 119         return ret;
 120 }
 121 #endif
 122
 123 /**
 124  * This is a simple portable iconv() implementaion.
 125  *
 126  * It only knows about a very small number of character sets - just
 127  * enough that Samba works on systems that don't have iconv.
 128  **/
 129 size_t smb_iconv(smb_iconv_t cd,
 130                  const char **inbuf, size_t *inbytesleft,
 131                  char **outbuf, size_t *outbytesleft)
 132 {
 133         char cvtbuf[2048];
 134         size_t bufsize;
 135
 136         /* in many cases we can go direct */
 137         if (cd->direct) {
 138                 return cd->direct(cd->cd_direct,
 139                                   inbuf, inbytesleft, outbuf, outbytesleft);
 140         }
 141
 142
 143         /* otherwise we have to do it chunks at a time */
 144         while (*inbytesleft > 0) {
 145                 char *bufp1 = cvtbuf;
 146                 const char *bufp2 = cvtbuf;
 147
 148                 bufsize = sizeof(cvtbuf);
 149
 150                 if (cd->pull(cd->cd_pull,
 151                              inbuf, inbytesleft, &bufp1, &bufsize) == -1
 152                     && errno != E2BIG) return -1;
 153
 154                 bufsize = sizeof(cvtbuf) - bufsize;
 155
 156                 if (cd->push(cd->cd_push,
 157                              &bufp2, &bufsize,
 158                              outbuf, outbytesleft) == -1) return -1;
 159         }
 160
 161         return 0;
 162 }
 163
 164 static BOOL is_utf16(const char *name)
 165 {
 166         return strcasecmp(name, "UCS-2LE") == 0 ||
 167                 strcasecmp(name, "UTF-16LE") == 0;
 168 }
 169
 170 /*
 171   simple iconv_open() wrapper
 172  */
 173 smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
 174 {
 175         smb_iconv_t ret;
 176         struct charset_functions *from, *to;
 177
 178         lazy_initialize_iconv();
 179         from = charsets;
 180         to = charsets;
 181
 182         ret = (smb_iconv_t)talloc_named(NULL, sizeof(*ret),
 183                                         "iconv(%s,%s)", tocode, fromcode);
 184         if (!ret) {
 185                 errno = ENOMEM;
 186                 return (smb_iconv_t)-1;
 187         }
 188         memset(ret, 0, sizeof(*ret));
 189
 190         /* check for the simplest null conversion */
 191         if (strcmp(fromcode, tocode) == 0) {
 192                 ret->direct = iconv_copy;
 193                 return ret;
 194         }
 195
 196         while (from) {
 197                 if (strcasecmp(from->name, fromcode) == 0) break;
 198                 from = from->next;
 199         }
 200
 201         while (to) {
 202                 if (strcasecmp(to->name, tocode) == 0) break;
 203                 to = to->next;
 204         }
 205
 206 #ifdef HAVE_NATIVE_ICONV
 207         if (!from) {
 208                 ret->pull = sys_iconv;
 209                 ret->cd_pull = iconv_open("UTF-16LE", fromcode);
 210                 if (ret->cd_pull == (iconv_t)-1)
 211                         ret->cd_pull = iconv_open("UCS-2LE", fromcode);
 212                 if (ret->cd_pull == (iconv_t)-1) goto failed;
 213         }
 214
 215         if (!to) {
 216                 ret->push = sys_iconv;
 217                 ret->cd_push = iconv_open(tocode, "UTF-16LE");
 218                 if (ret->cd_push == (iconv_t)-1)
 219                         ret->cd_push = iconv_open(tocode, "UCS-2LE");
 220                 if (ret->cd_push == (iconv_t)-1) goto failed;
 221         }
 222 #else
 223         if (!from || !to) {
 224                 goto failed;
 225         }
 226 #endif
 227
 228         /* check for conversion to/from ucs2 */
 229         if (is_utf16(fromcode) && to) {
 230                 ret->direct = to->push;
 231                 return ret;
 232         }
 233         if (is_utf16(tocode) && from) {
 234                 ret->direct = from->pull;
 235                 return ret;
 236         }
 237
 238 #ifdef HAVE_NATIVE_ICONV
 239         if (is_utf16(fromcode)) {
 240                 ret->direct = sys_iconv;
 241                 ret->cd_direct = ret->cd_push;
 242                 ret->cd_push = NULL;
 243                 return ret;
 244         }
 245         if (is_utf16(tocode)) {
 246                 ret->direct = sys_iconv;
 247                 ret->cd_direct = ret->cd_pull;
 248                 ret->cd_pull = NULL;
 249                 return ret;
 250         }
 251 #endif
 252
 253         /* the general case has to go via a buffer */
 254         if (!ret->pull) ret->pull = from->pull;
 255         if (!ret->push) ret->push = to->push;
 256         return ret;
 257
 258 failed:
 259         talloc_free(ret);
 260         errno = EINVAL;
 261         return (smb_iconv_t)-1;
 262 }
 263
 264 /*
 265   simple iconv_close() wrapper
 266 */
 267 int smb_iconv_close(smb_iconv_t cd)
 268 {
 269 #ifdef HAVE_NATIVE_ICONV
 270         if (cd->cd_direct) iconv_close((iconv_t)cd->cd_direct);
 271         if (cd->cd_pull) iconv_close((iconv_t)cd->cd_pull);
 272         if (cd->cd_push) iconv_close((iconv_t)cd->cd_push);
 273 #endif
 274
 275         talloc_free(cd);
 276         return 0;
 277 }
 278
 279
 280 /**********************************************************************
 281  the following functions implement the builtin character sets in Samba
 282  and also the "test" character sets that are designed to test
 283  multi-byte character set support for english users
 284 ***********************************************************************/
 285 static size_t ascii_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 286                          char **outbuf, size_t *outbytesleft)
 287 {
 288         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 289                 (*outbuf)[0] = (*inbuf)[0];
 290                 (*outbuf)[1] = 0;
 291                 (*inbytesleft)  -= 1;
 292                 (*outbytesleft) -= 2;
 293                 (*inbuf)  += 1;
 294                 (*outbuf) += 2;
 295         }
 296
 297         if (*inbytesleft > 0) {
 298                 errno = E2BIG;
 299                 return -1;
 300         }
 301
 302         return 0;
 303 }
 304
 305 static size_t ascii_push(void *cd, const char **inbuf, size_t *inbytesleft,
 306                          char **outbuf, size_t *outbytesleft)
 307 {
 308         int ir_count=0;
 309
 310         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 311                 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
 312                 if ((*inbuf)[1]) ir_count++;
 313                 (*inbytesleft)  -= 2;
 314                 (*outbytesleft) -= 1;
 315                 (*inbuf)  += 2;
 316                 (*outbuf) += 1;
 317         }
 318
 319         if (*inbytesleft == 1) {
 320                 errno = EINVAL;
 321                 return -1;
 322         }
 323
 324         if (*inbytesleft > 1) {
 325                 errno = E2BIG;
 326                 return -1;
 327         }
 328
 329         return ir_count;
 330 }
 331
 332
 333 static size_t ucs2hex_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 334                          char **outbuf, size_t *outbytesleft)
 335 {
 336         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 337                 uint_t v;
 338
 339                 if ((*inbuf)[0] != '@') {
 340                         /* seven bit ascii case */
 341                         (*outbuf)[0] = (*inbuf)[0];
 342                         (*outbuf)[1] = 0;
 343                         (*inbytesleft)  -= 1;
 344                         (*outbytesleft) -= 2;
 345                         (*inbuf)  += 1;
 346                         (*outbuf) += 2;
 347                         continue;
 348                 }
 349                 /* it's a hex character */
 350                 if (*inbytesleft < 5) {
 351                         errno = EINVAL;
 352                         return -1;
 353                 }
 354
 355                 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
 356                         errno = EILSEQ;
 357                         return -1;
 358                 }
 359
 360                 (*outbuf)[0] = v&0xff;
 361                 (*outbuf)[1] = v>>8;
 362                 (*inbytesleft)  -= 5;
 363                 (*outbytesleft) -= 2;
 364                 (*inbuf)  += 5;
 365                 (*outbuf) += 2;
 366         }
 367
 368         if (*inbytesleft > 0) {
 369                 errno = E2BIG;
 370                 return -1;
 371         }
 372
 373         return 0;
 374 }
 375
 376 static size_t ucs2hex_push(void *cd, const char **inbuf, size_t *inbytesleft,
 377                            char **outbuf, size_t *outbytesleft)
 378 {
 379         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 380                 char buf[6];
 381
 382                 if ((*inbuf)[1] == 0 &&
 383                     ((*inbuf)[0] & 0x80) == 0 &&
 384                     (*inbuf)[0] != '@') {
 385                         (*outbuf)[0] = (*inbuf)[0];
 386                         (*inbytesleft)  -= 2;
 387                         (*outbytesleft) -= 1;
 388                         (*inbuf)  += 2;
 389                         (*outbuf) += 1;
 390                         continue;
 391                 }
 392                 if (*outbytesleft < 5) {
 393                         errno = E2BIG;
 394                         return -1;
 395                 }
 396                 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
 397                 memcpy(*outbuf, buf, 5);
 398                 (*inbytesleft)  -= 2;
 399                 (*outbytesleft) -= 5;
 400                 (*inbuf)  += 2;
 401                 (*outbuf) += 5;
 402         }
 403
 404         if (*inbytesleft == 1) {
 405                 errno = EINVAL;
 406                 return -1;
 407         }
 408
 409         if (*inbytesleft > 1) {
 410                 errno = E2BIG;
 411                 return -1;
 412         }
 413
 414         return 0;
 415 }
 416
 417 static size_t iconv_swab(void *cd, const char **inbuf, size_t *inbytesleft,
 418                          char **outbuf, size_t *outbytesleft)
 419 {
 420         int n;
 421
 422         n = MIN(*inbytesleft, *outbytesleft);
 423
 424         swab(*inbuf, *outbuf, (n&~1));
 425         if (n&1) {
 426                 (*outbuf)[n-1] = 0;
 427         }
 428
 429         (*inbytesleft) -= n;
 430         (*outbytesleft) -= n;
 431         (*inbuf) += n;
 432         (*outbuf) += n;
 433
 434         if (*inbytesleft > 0) {
 435                 errno = E2BIG;
 436                 return -1;
 437         }
 438
 439         return 0;
 440 }
 441
 442
 443 static size_t iconv_copy(void *cd, const char **inbuf, size_t *inbytesleft,
 444                          char **outbuf, size_t *outbytesleft)
 445 {
 446         int n;
 447
 448         n = MIN(*inbytesleft, *outbytesleft);
 449
 450         memmove(*outbuf, *inbuf, n);
 451
 452         (*inbytesleft) -= n;
 453         (*outbytesleft) -= n;
 454         (*inbuf) += n;
 455         (*outbuf) += n;
 456
 457         if (*inbytesleft > 0) {
 458                 errno = E2BIG;
 459                 return -1;
 460         }
 461
 462         return 0;
 463 }
 464
 465 static size_t utf8_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 466                          char **outbuf, size_t *outbytesleft)
 467 {
 468         size_t in_left=*inbytesleft, out_left=*outbytesleft;
 469         const uint8_t *c = (const uint8_t *)*inbuf;
 470         uint8_t *uc = (uint8_t *)*outbuf;
 471
 472         while (in_left >= 1 && out_left >= 2) {
 473                 if ((c[0] & 0x80) == 0) {
 474                         uc[0] = c[0];
 475                         uc[1] = 0;
 476                         c  += 1;
 477                         in_left  -= 1;
 478                         out_left -= 2;
 479                         uc += 2;
 480                         continue;
 481                 }
 482
 483                 if ((c[0] & 0xe0) == 0xc0) {
 484                         if (in_left < 2 ||
 485                             (c[1] & 0xc0) != 0x80) {
 486                                 errno = EILSEQ;
 487                                 goto error;
 488                         }
 489                         uc[1] = (c[0]>>2) & 0x7;
 490                         uc[0] = (c[0]<<6) | (c[1]&0x3f);
 491                         c  += 2;
 492                         in_left  -= 2;
 493                         out_left -= 2;
 494                         uc += 2;
 495                         continue;
 496                 }
 497
 498                 if ((c[0] & 0xf0) == 0xe0) {
 499                         if (in_left < 3 ||
 500                             (c[1] & 0xc0) != 0x80 ||
 501                             (c[2] & 0xc0) != 0x80) {
 502                                 errno = EILSEQ;
 503                                 goto error;
 504                         }
 505                         uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
 506                         uc[0] = (c[1]<<6) | (c[2]&0x3f);
 507                         c  += 3;
 508                         in_left  -= 3;
 509                         out_left -= 2;
 510                         uc += 2;
 511                         continue;
 512                 }
 513
 514                 if ((c[0] & 0xf8) == 0xf0) {
 515                         unsigned int codepoint;
 516                         if (in_left < 4 ||
 517                             (c[1] & 0xc0) != 0x80 ||
 518                             (c[2] & 0xc0) != 0x80 ||
 519                             (c[3] & 0xc0) != 0x80) {
 520                                 errno = EILSEQ;
 521                                 goto error;
 522                         }
 523                         codepoint =
 524                                 (c[3]&0x3f) |
 525                                 ((c[2]&0x3f)<<6) |
 526                                 ((c[1]&0x3f)<<12) |
 527                                 ((c[0]&0x7)<<18);
 528                         if (codepoint < 0x10000) {
 529                                 /* accept UTF-8 characters that are not
 530                                    minimally packed, but pack the result */
 531                                 uc[0] = (codepoint & 0xFF);
 532                                 uc[1] = (codepoint >> 8);
 533                                 c += 4;
 534                                 in_left -= 4;
 535                                 out_left -= 2;
 536                                 uc += 2;
 537                                 continue;
 538                         }
 539
 540                         codepoint -= 0x10000;
 541
 542                         if (out_left < 4) {
 543                                 errno = E2BIG;
 544                                 goto error;
 545                         }
 546
 547                         uc[0] = (codepoint>>10) & 0xFF;
 548                         uc[1] = (codepoint>>18) | 0xd8;
 549                         uc[2] = codepoint & 0xFF;
 550                         uc[3] = ((codepoint>>8) & 0x3) | 0xdc;
 551                         c  += 4;
 552                         in_left  -= 4;
 553                         out_left -= 4;
 554                         uc += 4;
 555                         continue;
 556                 }
 557
 558                 /* we don't handle 5 byte sequences */
 559                 errno = EINVAL;
 560                 goto error;
 561         }
 562
 563         if (in_left > 0) {
 564                 errno = E2BIG;
 565                 goto error;
 566         }
 567
 568         *inbytesleft = in_left;
 569         *outbytesleft = out_left;
 570         *inbuf = c;
 571         *outbuf = uc;
 572         return 0;
 573
 574 error:
 575         *inbytesleft = in_left;
 576         *outbytesleft = out_left;
 577         *inbuf = c;
 578         *outbuf = uc;
 579         return -1;
 580 }
 581
 582 static size_t utf8_push(void *cd, const char **inbuf, size_t *inbytesleft,
 583                         char **outbuf, size_t *outbytesleft)
 584 {
 585         size_t in_left=*inbytesleft, out_left=*outbytesleft;
 586         uint8_t *c = (uint8_t *)*outbuf;
 587         const uint8_t *uc = (const uint8_t *)*inbuf;
 588
 589         while (in_left >= 2 && out_left >= 1) {
 590                 unsigned int codepoint;
 591
 592                 if (uc[1] == 0 && !(uc[0] & 0x80)) {
 593                         /* simplest case */
 594                         c[0] = uc[0];
 595                         in_left  -= 2;
 596                         out_left -= 1;
 597                         uc += 2;
 598                         c  += 1;
 599                         continue;
 600                 }
 601
 602                 if ((uc[1]&0xf8) == 0) {
 603                         /* next simplest case */
 604                         if (out_left < 2) {
 605                                 errno = E2BIG;
 606                                 goto error;
 607                         }
 608                         c[0] = 0xc0 | (uc[0]>>6) | (uc[1]<<2);
 609                         c[1] = 0x80 | (uc[0] & 0x3f);
 610                         in_left  -= 2;
 611                         out_left -= 2;
 612                         uc += 2;
 613                         c  += 2;
 614                         continue;
 615                 }
 616
 617                 if ((uc[1] & 0xfc) == 0xdc) {
 618                         /* its the second part of a 4 byte sequence. Illegal */
 619                         if (in_left < 4) {
 620                                 errno = EINVAL;
 621                         } else {
 622                                 errno = EILSEQ;
 623                         }
 624                         goto error;
 625                 }
 626
 627                 if ((uc[1] & 0xfc) != 0xd8) {
 628                         codepoint = uc[0] | (uc[1]<<8);
 629                         if (out_left < 3) {
 630                                 errno = E2BIG;
 631                                 goto error;
 632                         }
 633                         c[0] = 0xe0 | (codepoint >> 12);
 634                         c[1] = 0x80 | ((codepoint >> 6) & 0x3f);
 635                         c[2] = 0x80 | (codepoint & 0x3f);
 636
 637                         in_left  -= 2;
 638                         out_left -= 3;
 639                         uc  += 2;
 640                         c   += 3;
 641                         continue;
 642                 }
 643
 644                 /* its the first part of a 4 byte sequence */
 645                 if (in_left < 4) {
 646                         errno = EINVAL;
 647                         goto error;
 648                 }
 649                 if ((uc[3] & 0xfc) != 0xdc) {
 650                         errno = EILSEQ;
 651                         goto error;
 652                 }
 653                 codepoint = 0x10000 + (uc[2] | ((uc[3] & 0x3)<<8) |
 654                                        (uc[0]<<10) | ((uc[1] & 0x3)<<18));
 655
 656                 if (out_left < 4) {
 657                         errno = E2BIG;
 658                         goto error;
 659                 }
 660                 c[0] = 0xf0 | (codepoint >> 18);
 661                 c[1] = 0x80 | ((codepoint >> 12) & 0x3f);
 662                 c[2] = 0x80 | ((codepoint >> 6) & 0x3f);
 663                 c[3] = 0x80 | (codepoint & 0x3f);
 664
 665                 in_left  -= 4;
 666                 out_left -= 4;
 667                 uc       += 4;
 668                 c        += 4;
 669         }
 670
 671         if (in_left == 1) {
 672                 errno = EINVAL;
 673                 goto error;
 674         }
 675
 676         if (in_left > 1) {
 677                 errno = E2BIG;
 678                 goto error;
 679         }
 680
 681         *inbytesleft = in_left;
 682         *outbytesleft = out_left;
 683         *inbuf  = uc;
 684         *outbuf = c;
 685
 686         return 0;
 687
 688 error:
 689         *inbytesleft = in_left;
 690         *outbytesleft = out_left;
 691         *inbuf  = uc;
 692         *outbuf = c;
 693         return -1;
 694 }
 695
 696
 697