lib/util/charset/iconv.c

   1 /*
   2    Unix SMB/CIFS implementation.
   3    minimal iconv implementation
   4    Copyright (C) Andrew Tridgell 2001
   5    Copyright (C) Jelmer Vernooij 2002
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 */
  20
  21 #include "includes.h"
  22 #include "../lib/util/dlinklist.h"
  23 #include "system/iconv.h"
  24 #include "system/filesys.h"
  25
  26
  27 /**
  28  * @file
  29  *
  30  * @brief Samba wrapper/stub for iconv character set conversion.
  31  *
  32  * iconv is the XPG2 interface for converting between character
  33  * encodings.  This file provides a Samba wrapper around it, and also
  34  * a simple reimplementation that is used if the system does not
  35  * implement iconv.
  36  *
  37  * Samba only works with encodings that are supersets of ASCII: ascii
  38  * characters like whitespace can be tested for directly, multibyte
  39  * sequences start with a byte with the high bit set, and strings are
  40  * terminated by a nul byte.
  41  *
  42  * Note that the only function provided by iconv is conversion between
  43  * characters.  It doesn't directly support operations like
  44  * uppercasing or comparison.  We have to convert to UTF-16LE and
  45  * compare there.
  46  *
  47  * @sa Samba Developers Guide
  48  **/
  49
  50 static size_t ascii_pull  (void *,const char **, size_t *, char **, size_t *);
  51 static size_t ascii_push  (void *,const char **, size_t *, char **, size_t *);
  52 static size_t utf8_pull   (void *,const char **, size_t *, char **, size_t *);
  53 static size_t utf8_push   (void *,const char **, size_t *, char **, size_t *);
  54 static size_t utf8_munged_push(void *,const char **, size_t *, char **, size_t *);
  55 static size_t ucs2hex_pull(void *,const char **, size_t *, char **, size_t *);
  56 static size_t ucs2hex_push(void *,const char **, size_t *, char **, size_t *);
  57 static size_t iconv_copy  (void *,const char **, size_t *, char **, size_t *);
  58 static size_t iconv_swab  (void *,const char **, size_t *, char **, size_t *);
  59
  60 static const struct charset_functions builtin_functions[] = {
  61         /* windows is closest to UTF-16 */
  62         {"UCS-2LE",  iconv_copy, iconv_copy},
  63         {"UTF-16LE",  iconv_copy, iconv_copy},
  64         {"UCS-2BE",  iconv_swab, iconv_swab},
  65         {"UTF-16BE",  iconv_swab, iconv_swab},
  66
  67         /* we include the UTF-8 alias to cope with differing locale settings */
  68         {"UTF8",   utf8_pull,  utf8_push},
  69         {"UTF-8",   utf8_pull,  utf8_push},
  70
  71         /* this handles the munging needed for String2Key */
  72         {"UTF8_MUNGED",   utf8_pull,  utf8_munged_push},
  73
  74         {"ASCII", ascii_pull, ascii_push},
  75         {"UCS2-HEX", ucs2hex_pull, ucs2hex_push}
  76 };
  77
  78 static struct charset_functions *charsets = NULL;
  79
  80 bool charset_register_backend(const void *_funcs)
  81 {
  82         struct charset_functions *funcs = (struct charset_functions *)memdup(_funcs,sizeof(struct charset_functions));
  83         struct charset_functions *c;
  84
  85         /* Check whether we already have this charset... */
  86         for (c = charsets; c != NULL; c = c->next) {
  87                 if(!strcasecmp(c->name, funcs->name)) {
  88                         DEBUG(2, ("Duplicate charset %s, not registering\n", funcs->name));
  89                         return false;
  90                 }
  91         }
  92
  93         funcs->next = funcs->prev = NULL;
  94         DLIST_ADD(charsets, funcs);
  95         return true;
  96 }
  97
  98 #ifdef HAVE_NATIVE_ICONV
  99 /* if there was an error then reset the internal state,
 100    this ensures that we don't have a shift state remaining for
 101    character sets like SJIS */
 102 static size_t sys_iconv(void *cd,
 103                         const char **inbuf, size_t *inbytesleft,
 104                         char **outbuf, size_t *outbytesleft)
 105 {
 106         size_t ret = iconv((iconv_t)cd,
 107                            discard_const_p(char *, inbuf), inbytesleft,
 108                            outbuf, outbytesleft);
 109         if (ret == (size_t)-1) iconv(cd, NULL, NULL, NULL, NULL);
 110         return ret;
 111 }
 112 #endif
 113
 114 /**
 115  * This is a simple portable iconv() implementaion.
 116  *
 117  * It only knows about a very small number of character sets - just
 118  * enough that Samba works on systems that don't have iconv.
 119  **/
 120 _PUBLIC_ size_t smb_iconv(smb_iconv_t cd,
 121                  const char **inbuf, size_t *inbytesleft,
 122                  char **outbuf, size_t *outbytesleft)
 123 {
 124         char cvtbuf[2048];
 125         size_t bufsize;
 126
 127         /* in many cases we can go direct */
 128         if (cd->direct) {
 129                 return cd->direct(cd->cd_direct,
 130                                   inbuf, inbytesleft, outbuf, outbytesleft);
 131         }
 132
 133
 134         /* otherwise we have to do it chunks at a time */
 135         while (*inbytesleft > 0) {
 136                 char *bufp1 = cvtbuf;
 137                 const char *bufp2 = cvtbuf;
 138
 139                 bufsize = sizeof(cvtbuf);
 140
 141                 if (cd->pull(cd->cd_pull,
 142                              inbuf, inbytesleft, &bufp1, &bufsize) == -1
 143                     && errno != E2BIG) return -1;
 144
 145                 bufsize = sizeof(cvtbuf) - bufsize;
 146
 147                 if (cd->push(cd->cd_push,
 148                              &bufp2, &bufsize,
 149                              outbuf, outbytesleft) == -1) return -1;
 150         }
 151
 152         return 0;
 153 }
 154
 155 static bool is_utf16(const char *name)
 156 {
 157         return strcasecmp(name, "UCS-2LE") == 0 ||
 158                 strcasecmp(name, "UTF-16LE") == 0;
 159 }
 160
 161
 162
 163 _PUBLIC_ smb_iconv_t smb_iconv_open_ex(TALLOC_CTX *mem_ctx, const char *tocode,
 164                               const char *fromcode, bool native_iconv)
 165 {
 166         smb_iconv_t ret;
 167         const struct charset_functions *from=NULL, *to=NULL;
 168         int i;
 169
 170         ret = (smb_iconv_t)talloc_named(mem_ctx,
 171                                         sizeof(*ret),
 172                                         "iconv(%s,%s)", tocode, fromcode);
 173         if (!ret) {
 174                 errno = ENOMEM;
 175                 return (smb_iconv_t)-1;
 176         }
 177         memset(ret, 0, sizeof(*ret));
 178
 179         /* check for the simplest null conversion */
 180         if (strcmp(fromcode, tocode) == 0) {
 181                 ret->direct = iconv_copy;
 182                 return ret;
 183         }
 184
 185         for (i=0;i<ARRAY_SIZE(builtin_functions);i++) {
 186                 if (strcasecmp(fromcode, builtin_functions[i].name) == 0) {
 187                         from = &builtin_functions[i];
 188                 }
 189                 if (strcasecmp(tocode, builtin_functions[i].name) == 0) {
 190                         to = &builtin_functions[i];
 191                 }
 192         }
 193
 194         if (from == NULL) {
 195                 for (from=charsets; from; from=from->next) {
 196                         if (strcasecmp(from->name, fromcode) == 0) break;
 197                 }
 198         }
 199
 200         if (to == NULL) {
 201                 for (to=charsets; to; to=to->next) {
 202                         if (strcasecmp(to->name, tocode) == 0) break;
 203                 }
 204         }
 205
 206 #ifdef HAVE_NATIVE_ICONV
 207         if ((!from || !to) && !native_iconv) {
 208                 goto failed;
 209         }
 210         if (!from) {
 211                 ret->pull = sys_iconv;
 212                 ret->cd_pull = iconv_open("UTF-16LE", fromcode);
 213                 if (ret->cd_pull == (iconv_t)-1)
 214                         ret->cd_pull = iconv_open("UCS-2LE", fromcode);
 215                 if (ret->cd_pull == (iconv_t)-1) goto failed;
 216         }
 217
 218         if (!to) {
 219                 ret->push = sys_iconv;
 220                 ret->cd_push = iconv_open(tocode, "UTF-16LE");
 221                 if (ret->cd_push == (iconv_t)-1)
 222                         ret->cd_push = iconv_open(tocode, "UCS-2LE");
 223                 if (ret->cd_push == (iconv_t)-1) goto failed;
 224         }
 225 #else
 226         if (!from || !to) {
 227                 goto failed;
 228         }
 229 #endif
 230
 231         /* check for conversion to/from ucs2 */
 232         if (is_utf16(fromcode) && to) {
 233                 ret->direct = to->push;
 234                 return ret;
 235         }
 236         if (is_utf16(tocode) && from) {
 237                 ret->direct = from->pull;
 238                 return ret;
 239         }
 240
 241 #ifdef HAVE_NATIVE_ICONV
 242         if (is_utf16(fromcode)) {
 243                 ret->direct = sys_iconv;
 244                 ret->cd_direct = ret->cd_push;
 245                 ret->cd_push = NULL;
 246                 return ret;
 247         }
 248         if (is_utf16(tocode)) {
 249                 ret->direct = sys_iconv;
 250                 ret->cd_direct = ret->cd_pull;
 251                 ret->cd_pull = NULL;
 252                 return ret;
 253         }
 254 #endif
 255
 256         /* the general case has to go via a buffer */
 257         if (!ret->pull) ret->pull = from->pull;
 258         if (!ret->push) ret->push = to->push;
 259         return ret;
 260
 261 failed:
 262         talloc_free(ret);
 263         errno = EINVAL;
 264         return (smb_iconv_t)-1;
 265 }
 266
 267 /*
 268   simple iconv_open() wrapper
 269  */
 270 _PUBLIC_ smb_iconv_t smb_iconv_open(const char *tocode, const char *fromcode)
 271 {
 272         return smb_iconv_open_ex(NULL, tocode, fromcode, true);
 273 }
 274
 275 /*
 276   simple iconv_close() wrapper
 277 */
 278 _PUBLIC_ int smb_iconv_close(smb_iconv_t cd)
 279 {
 280 #ifdef HAVE_NATIVE_ICONV
 281         if (cd->cd_direct) iconv_close((iconv_t)cd->cd_direct);
 282         if (cd->cd_pull) iconv_close((iconv_t)cd->cd_pull);
 283         if (cd->cd_push) iconv_close((iconv_t)cd->cd_push);
 284 #endif
 285
 286         talloc_free(cd);
 287         return 0;
 288 }
 289
 290
 291 /**********************************************************************
 292  the following functions implement the builtin character sets in Samba
 293  and also the "test" character sets that are designed to test
 294  multi-byte character set support for english users
 295 ***********************************************************************/
 296 static size_t ascii_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 297                          char **outbuf, size_t *outbytesleft)
 298 {
 299         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 300                 (*outbuf)[0] = (*inbuf)[0];
 301                 (*outbuf)[1] = 0;
 302                 (*inbytesleft)  -= 1;
 303                 (*outbytesleft) -= 2;
 304                 (*inbuf)  += 1;
 305                 (*outbuf) += 2;
 306         }
 307
 308         if (*inbytesleft > 0) {
 309                 errno = E2BIG;
 310                 return -1;
 311         }
 312
 313         return 0;
 314 }
 315
 316 static size_t ascii_push(void *cd, const char **inbuf, size_t *inbytesleft,
 317                          char **outbuf, size_t *outbytesleft)
 318 {
 319         int ir_count=0;
 320
 321         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 322                 (*outbuf)[0] = (*inbuf)[0] & 0x7F;
 323                 if ((*inbuf)[1]) ir_count++;
 324                 (*inbytesleft)  -= 2;
 325                 (*outbytesleft) -= 1;
 326                 (*inbuf)  += 2;
 327                 (*outbuf) += 1;
 328         }
 329
 330         if (*inbytesleft == 1) {
 331                 errno = EINVAL;
 332                 return -1;
 333         }
 334
 335         if (*inbytesleft > 1) {
 336                 errno = E2BIG;
 337                 return -1;
 338         }
 339
 340         return ir_count;
 341 }
 342
 343
 344 static size_t ucs2hex_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 345                          char **outbuf, size_t *outbytesleft)
 346 {
 347         while (*inbytesleft >= 1 && *outbytesleft >= 2) {
 348                 uint_t v;
 349
 350                 if ((*inbuf)[0] != '@') {
 351                         /* seven bit ascii case */
 352                         (*outbuf)[0] = (*inbuf)[0];
 353                         (*outbuf)[1] = 0;
 354                         (*inbytesleft)  -= 1;
 355                         (*outbytesleft) -= 2;
 356                         (*inbuf)  += 1;
 357                         (*outbuf) += 2;
 358                         continue;
 359                 }
 360                 /* it's a hex character */
 361                 if (*inbytesleft < 5) {
 362                         errno = EINVAL;
 363                         return -1;
 364                 }
 365
 366                 if (sscanf(&(*inbuf)[1], "%04x", &v) != 1) {
 367                         errno = EILSEQ;
 368                         return -1;
 369                 }
 370
 371                 (*outbuf)[0] = v&0xff;
 372                 (*outbuf)[1] = v>>8;
 373                 (*inbytesleft)  -= 5;
 374                 (*outbytesleft) -= 2;
 375                 (*inbuf)  += 5;
 376                 (*outbuf) += 2;
 377         }
 378
 379         if (*inbytesleft > 0) {
 380                 errno = E2BIG;
 381                 return -1;
 382         }
 383
 384         return 0;
 385 }
 386
 387 static size_t ucs2hex_push(void *cd, const char **inbuf, size_t *inbytesleft,
 388                            char **outbuf, size_t *outbytesleft)
 389 {
 390         while (*inbytesleft >= 2 && *outbytesleft >= 1) {
 391                 char buf[6];
 392
 393                 if ((*inbuf)[1] == 0 &&
 394                     ((*inbuf)[0] & 0x80) == 0 &&
 395                     (*inbuf)[0] != '@') {
 396                         (*outbuf)[0] = (*inbuf)[0];
 397                         (*inbytesleft)  -= 2;
 398                         (*outbytesleft) -= 1;
 399                         (*inbuf)  += 2;
 400                         (*outbuf) += 1;
 401                         continue;
 402                 }
 403                 if (*outbytesleft < 5) {
 404                         errno = E2BIG;
 405                         return -1;
 406                 }
 407                 snprintf(buf, 6, "@%04x", SVAL(*inbuf, 0));
 408                 memcpy(*outbuf, buf, 5);
 409                 (*inbytesleft)  -= 2;
 410                 (*outbytesleft) -= 5;
 411                 (*inbuf)  += 2;
 412                 (*outbuf) += 5;
 413         }
 414
 415         if (*inbytesleft == 1) {
 416                 errno = EINVAL;
 417                 return -1;
 418         }
 419
 420         if (*inbytesleft > 1) {
 421                 errno = E2BIG;
 422                 return -1;
 423         }
 424
 425         return 0;
 426 }
 427
 428 static size_t iconv_swab(void *cd, const char **inbuf, size_t *inbytesleft,
 429                          char **outbuf, size_t *outbytesleft)
 430 {
 431         int n;
 432
 433         n = MIN(*inbytesleft, *outbytesleft);
 434
 435         swab(*inbuf, *outbuf, (n&~1));
 436         if (n&1) {
 437                 (*outbuf)[n-1] = 0;
 438         }
 439
 440         (*inbytesleft) -= n;
 441         (*outbytesleft) -= n;
 442         (*inbuf) += n;
 443         (*outbuf) += n;
 444
 445         if (*inbytesleft > 0) {
 446                 errno = E2BIG;
 447                 return -1;
 448         }
 449
 450         return 0;
 451 }
 452
 453
 454 static size_t iconv_copy(void *cd, const char **inbuf, size_t *inbytesleft,
 455                          char **outbuf, size_t *outbytesleft)
 456 {
 457         int n;
 458
 459         n = MIN(*inbytesleft, *outbytesleft);
 460
 461         memmove(*outbuf, *inbuf, n);
 462
 463         (*inbytesleft) -= n;
 464         (*outbytesleft) -= n;
 465         (*inbuf) += n;
 466         (*outbuf) += n;
 467
 468         if (*inbytesleft > 0) {
 469                 errno = E2BIG;
 470                 return -1;
 471         }
 472
 473         return 0;
 474 }
 475
 476 /*
 477   this takes a UTF8 sequence and produces a UTF16 sequence
 478  */
 479 static size_t utf8_pull(void *cd, const char **inbuf, size_t *inbytesleft,
 480                          char **outbuf, size_t *outbytesleft)
 481 {
 482         size_t in_left=*inbytesleft, out_left=*outbytesleft;
 483         const uint8_t *c = (const uint8_t *)*inbuf;
 484         uint8_t *uc = (uint8_t *)*outbuf;
 485
 486         while (in_left >= 1 && out_left >= 2) {
 487                 if ((c[0] & 0x80) == 0) {
 488                         uc[0] = c[0];
 489                         uc[1] = 0;
 490                         c  += 1;
 491                         in_left  -= 1;
 492                         out_left -= 2;
 493                         uc += 2;
 494                         continue;
 495                 }
 496
 497                 if ((c[0] & 0xe0) == 0xc0) {
 498                         if (in_left < 2 ||
 499                             (c[1] & 0xc0) != 0x80) {
 500                                 errno = EILSEQ;
 501                                 goto error;
 502                         }
 503                         uc[1] = (c[0]>>2) & 0x7;
 504                         uc[0] = (c[0]<<6) | (c[1]&0x3f);
 505                         c  += 2;
 506                         in_left  -= 2;
 507                         out_left -= 2;
 508                         uc += 2;
 509                         continue;
 510                 }
 511
 512                 if ((c[0] & 0xf0) == 0xe0) {
 513                         if (in_left < 3 ||
 514                             (c[1] & 0xc0) != 0x80 ||
 515                             (c[2] & 0xc0) != 0x80) {
 516                                 errno = EILSEQ;
 517                                 goto error;
 518                         }
 519                         uc[1] = ((c[0]&0xF)<<4) | ((c[1]>>2)&0xF);
 520                         uc[0] = (c[1]<<6) | (c[2]&0x3f);
 521                         c  += 3;
 522                         in_left  -= 3;
 523                         out_left -= 2;
 524                         uc += 2;
 525                         continue;
 526                 }
 527
 528                 if ((c[0] & 0xf8) == 0xf0) {
 529                         unsigned int codepoint;
 530                         if (in_left < 4 ||
 531                             (c[1] & 0xc0) != 0x80 ||
 532                             (c[2] & 0xc0) != 0x80 ||
 533                             (c[3] & 0xc0) != 0x80) {
 534                                 errno = EILSEQ;
 535                                 goto error;
 536                         }
 537                         codepoint =
 538                                 (c[3]&0x3f) |
 539                                 ((c[2]&0x3f)<<6) |
 540                                 ((c[1]&0x3f)<<12) |
 541                                 ((c[0]&0x7)<<18);
 542                         if (codepoint < 0x10000) {
 543                                 /* accept UTF-8 characters that are not
 544                                    minimally packed, but pack the result */
 545                                 uc[0] = (codepoint & 0xFF);
 546                                 uc[1] = (codepoint >> 8);
 547                                 c += 4;
 548                                 in_left -= 4;
 549                                 out_left -= 2;
 550                                 uc += 2;
 551                                 continue;
 552                         }
 553
 554                         codepoint -= 0x10000;
 555
 556                         if (out_left < 4) {
 557                                 errno = E2BIG;
 558                                 goto error;
 559                         }
 560
 561                         uc[0] = (codepoint>>10) & 0xFF;
 562                         uc[1] = (codepoint>>18) | 0xd8;
 563                         uc[2] = codepoint & 0xFF;
 564                         uc[3] = ((codepoint>>8) & 0x3) | 0xdc;
 565                         c  += 4;
 566                         in_left  -= 4;
 567                         out_left -= 4;
 568                         uc += 4;
 569                         continue;
 570                 }
 571
 572                 /* we don't handle 5 byte sequences */
 573                 errno = EINVAL;
 574                 goto error;
 575         }
 576
 577         if (in_left > 0) {
 578                 errno = E2BIG;
 579                 goto error;
 580         }
 581
 582         *inbytesleft = in_left;
 583         *outbytesleft = out_left;
 584         *inbuf = (const char *)c;
 585         *outbuf = (char *)uc;
 586         return 0;
 587
 588 error:
 589         *inbytesleft = in_left;
 590         *outbytesleft = out_left;
 591         *inbuf = (const char *)c;
 592         *outbuf = (char *)uc;
 593         return -1;
 594 }
 595
 596
 597 /*
 598   this takes a UTF16 sequence and produces a UTF8 sequence
 599  */
 600 static size_t utf8_push(void *cd, const char **inbuf, size_t *inbytesleft,
 601                         char **outbuf, size_t *outbytesleft)
 602 {
 603         size_t in_left=*inbytesleft, out_left=*outbytesleft;
 604         uint8_t *c = (uint8_t *)*outbuf;
 605         const uint8_t *uc = (const uint8_t *)*inbuf;
 606
 607         while (in_left >= 2 && out_left >= 1) {
 608                 unsigned int codepoint;
 609
 610                 if (uc[1] == 0 && !(uc[0] & 0x80)) {
 611                         /* simplest case */
 612                         c[0] = uc[0];
 613                         in_left  -= 2;
 614                         out_left -= 1;
 615                         uc += 2;
 616                         c  += 1;
 617                         continue;
 618                 }
 619
 620                 if ((uc[1]&0xf8) == 0) {
 621                         /* next simplest case */
 622                         if (out_left < 2) {
 623                                 errno = E2BIG;
 624                                 goto error;
 625                         }
 626                         c[0] = 0xc0 | (uc[0]>>6) | (uc[1]<<2);
 627                         c[1] = 0x80 | (uc[0] & 0x3f);
 628                         in_left  -= 2;
 629                         out_left -= 2;
 630                         uc += 2;
 631                         c  += 2;
 632                         continue;
 633                 }
 634
 635                 if ((uc[1] & 0xfc) == 0xdc) {
 636                         /* its the second part of a 4 byte sequence. Illegal */
 637                         if (in_left < 4) {
 638                                 errno = EINVAL;
 639                         } else {
 640                                 errno = EILSEQ;
 641                         }
 642                         goto error;
 643                 }
 644
 645                 if ((uc[1] & 0xfc) != 0xd8) {
 646                         codepoint = uc[0] | (uc[1]<<8);
 647                         if (out_left < 3) {
 648                                 errno = E2BIG;
 649                                 goto error;
 650                         }
 651                         c[0] = 0xe0 | (codepoint >> 12);
 652                         c[1] = 0x80 | ((codepoint >> 6) & 0x3f);
 653                         c[2] = 0x80 | (codepoint & 0x3f);
 654
 655                         in_left  -= 2;
 656                         out_left -= 3;
 657                         uc  += 2;
 658                         c   += 3;
 659                         continue;
 660                 }
 661
 662                 /* its the first part of a 4 byte sequence */
 663                 if (in_left < 4) {
 664                         errno = EINVAL;
 665                         goto error;
 666                 }
 667                 if ((uc[3] & 0xfc) != 0xdc) {
 668                         errno = EILSEQ;
 669                         goto error;
 670                 }
 671                 codepoint = 0x10000 + (uc[2] | ((uc[3] & 0x3)<<8) |
 672                                        (uc[0]<<10) | ((uc[1] & 0x3)<<18));
 673
 674                 if (out_left < 4) {
 675                         errno = E2BIG;
 676                         goto error;
 677                 }
 678                 c[0] = 0xf0 | (codepoint >> 18);
 679                 c[1] = 0x80 | ((codepoint >> 12) & 0x3f);
 680                 c[2] = 0x80 | ((codepoint >> 6) & 0x3f);
 681                 c[3] = 0x80 | (codepoint & 0x3f);
 682
 683                 in_left  -= 4;
 684                 out_left -= 4;
 685                 uc       += 4;
 686                 c        += 4;
 687         }
 688
 689         if (in_left == 1) {
 690                 errno = EINVAL;
 691                 goto error;
 692         }
 693
 694         if (in_left > 1) {
 695                 errno = E2BIG;
 696                 goto error;
 697         }
 698
 699         *inbytesleft = in_left;
 700         *outbytesleft = out_left;
 701         *inbuf  = (const char *)uc;
 702         *outbuf = (char *)c;
 703
 704         return 0;
 705
 706 error:
 707         *inbytesleft = in_left;
 708         *outbytesleft = out_left;
 709         *inbuf  = (const char *)uc;
 710         *outbuf = (char *)c;
 711         return -1;
 712 }
 713
 714
 715 /*
 716   this takes a UTF16 sequence, munges it according to the string2key
 717   rules, and produces a UTF8 sequence
 718
 719 The rules are:
 720
 721     1) convert any instance of 0xD800 - 0xDBFF (high surrogate)
 722        without an immediately following 0xDC00 - 0x0xDFFF (low surrogate) to
 723        U+FFFD (OBJECT REPLACEMENT CHARACTER).
 724
 725     2) the same for any low surrogate that was not preceded by a high surrogate.
 726  */
 727 static size_t utf8_munged_push(void *cd, const char **inbuf, size_t *inbytesleft,
 728                                char **outbuf, size_t *outbytesleft)
 729 {
 730         size_t in_left=*inbytesleft, out_left=*outbytesleft;
 731         uint8_t *c = (uint8_t *)*outbuf;
 732         const uint8_t *uc = (const uint8_t *)*inbuf;
 733
 734         while (in_left >= 2 && out_left >= 1) {
 735                 unsigned int codepoint;
 736
 737                 if (uc[1] == 0 && !(uc[0] & 0x80)) {
 738                         /* simplest case */
 739                         c[0] = uc[0];
 740                         in_left  -= 2;
 741                         out_left -= 1;
 742                         uc += 2;
 743                         c  += 1;
 744                         continue;
 745                 }
 746
 747                 if ((uc[1]&0xf8) == 0) {
 748                         /* next simplest case */
 749                         if (out_left < 2) {
 750                                 errno = E2BIG;
 751                                 goto error;
 752                         }
 753                         c[0] = 0xc0 | (uc[0]>>6) | (uc[1]<<2);
 754                         c[1] = 0x80 | (uc[0] & 0x3f);
 755                         in_left  -= 2;
 756                         out_left -= 2;
 757                         uc += 2;
 758                         c  += 2;
 759                         continue;
 760                 }
 761
 762                 if ((uc[1] & 0xfc) == 0xdc) {
 763                         /* low surrogate not preceded by high surrogate
 764                            convert to 0xfffd */
 765                         codepoint = 0xfffd;
 766                         goto codepoint16;
 767                 }
 768
 769                 if ((uc[1] & 0xfc) != 0xd8) {
 770                         codepoint = uc[0] | (uc[1]<<8);
 771                         goto codepoint16;
 772                 }
 773
 774                 /* its the first part of a 4 byte sequence */
 775                 if (in_left < 4 || (uc[3] & 0xfc) != 0xdc) {
 776                         /* high surrogate not followed by low surrogate
 777                            convert to 0xfffd */
 778                         codepoint = 0xfffd;
 779                         goto codepoint16;
 780                 }
 781
 782                 codepoint = 0x10000 + (uc[2] | ((uc[3] & 0x3)<<8) |
 783                                        (uc[0]<<10) | ((uc[1] & 0x3)<<18));
 784
 785                 if (out_left < 4) {
 786                         errno = E2BIG;
 787                         goto error;
 788                 }
 789                 c[0] = 0xf0 | (codepoint >> 18);
 790                 c[1] = 0x80 | ((codepoint >> 12) & 0x3f);
 791                 c[2] = 0x80 | ((codepoint >> 6) & 0x3f);
 792                 c[3] = 0x80 | (codepoint & 0x3f);
 793
 794                 in_left  -= 4;
 795                 out_left -= 4;
 796                 uc       += 4;
 797                 c        += 4;
 798                 continue;
 799
 800         codepoint16:
 801                 if (out_left < 3) {
 802                         errno = E2BIG;
 803                         goto error;
 804                 }
 805                 c[0] = 0xe0 | (codepoint >> 12);
 806                 c[1] = 0x80 | ((codepoint >> 6) & 0x3f);
 807                 c[2] = 0x80 | (codepoint & 0x3f);
 808
 809                 in_left  -= 2;
 810                 out_left -= 3;
 811                 uc  += 2;
 812                 c   += 3;
 813                 continue;
 814         }
 815
 816         if (in_left == 1) {
 817                 errno = EINVAL;
 818                 goto error;
 819         }
 820
 821         if (in_left > 1) {
 822                 errno = E2BIG;
 823                 goto error;
 824         }
 825
 826         *inbytesleft = in_left;
 827         *outbytesleft = out_left;
 828         *inbuf  = (const char *)uc;
 829         *outbuf = (char *)c;
 830
 831         return 0;
 832
 833 error:
 834         *inbytesleft = in_left;
 835         *outbytesleft = out_left;
 836         *inbuf  = (const char *)uc;
 837         *outbuf = (char *)c;
 838         return -1;
 839 }
 840
 841
 842