source4/lib/appweb/ejs-2.0/ejs/ejsLex.c

   1 /*
   2  *      @file   ejsLex.c
   3  *      @brief  EJS Lexical Analyser
   4  *      @overview EJS lexical analyser. This implementes a lexical analyser
   5  *              for a subset of the JavaScript language.
   6  */
   7 /********************************* Copyright **********************************/
   8 /*
   9  *      @copy   default.g
  10  *
  11  *      Copyright (c) Mbedthis Software LLC, 2003-2006. All Rights Reserved.
  12  *      Portions Copyright (c) GoAhead Software, 1995-2000. All Rights Reserved.
  13  *
  14  *      This software is distributed under commercial and open source licenses.
  15  *      You may use the GPL open source license described below or you may acquire
  16  *      a commercial license from Mbedthis Software. You agree to be fully bound
  17  *      by the terms of either license. Consult the LICENSE.TXT distributed with
  18  *      this software for full details.
  19  *
  20  *      This software is open source; you can redistribute it and/or modify it
  21  *      under the terms of the GNU General Public License as published by the
  22  *      Free Software Foundation; either version 2 of the License, or (at your
  23  *      option) any later version. See the GNU General Public License for more
  24  *      details at: http://www.mbedthis.com/downloads/gplLicense.html
  25  *
  26  *      This program is distributed WITHOUT ANY WARRANTY; without even the
  27  *      implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  28  *
  29  *      This GPL license does NOT permit incorporating this software into
  30  *      proprietary programs. If you are unable to comply with the GPL, you must
  31  *      acquire a commercial license to use this software. Commercial licenses
  32  *      for this software and support services are available from Mbedthis
  33  *      Software at http://www.mbedthis.com
  34  *
  35  *      @end
  36  */
  37 /********************************** Includes **********************************/
  38
  39 #include        "ejs.h"
  40
  41 #if BLD_FEATURE_EJS
  42
  43 /****************************** Forward Declarations **************************/
  44
  45 static int              getLexicalToken(Ejs *ep, int state);
  46 static int              tokenAddChar(Ejs *ep, int c);
  47 static int              inputGetc(Ejs *ep);
  48 static void             inputPutback(Ejs *ep, int c);
  49 static int              charConvert(Ejs *ep, int base, int maxDig);
  50 static void     parseNumber(Ejs *ep, EjsType type);
  51
  52 /************************************* Code ***********************************/
  53 /*
  54  *      Open a new input script
  55  */
  56
  57 int ejsLexOpenScript(Ejs *ep, const char *script)
  58 {
  59         EjsInput        *ip;
  60
  61         mprAssert(ep);
  62         mprAssert(script);
  63
  64         if ((ip = mprAllocTypeZeroed(ep, EjsInput)) == NULL) {
  65                 return MPR_ERR_MEMORY;
  66         }
  67         ip->next = ep->input;
  68         ep->input = ip;
  69         ip->procName = ep->proc ? ep->proc->procName : NULL;
  70         ip->fileName = ep->fileName ? ep->fileName : NULL;
  71
  72 /*
  73  *      Create the parse token buffer and script buffer
  74  */
  75         ip->tokServp = ip->tokbuf;
  76         ip->tokEndp = ip->tokbuf;
  77
  78         ip->script = script;
  79         ip->scriptSize = strlen(script);
  80         ip->scriptServp = (char*) ip->script;
  81
  82         ip->lineNumber = 1;
  83         ip->lineColumn = 0;
  84
  85         ip->putBackIndex = -1;
  86
  87         return 0;
  88 }
  89
  90 /******************************************************************************/
  91 /*
  92  *      Close the input script
  93  */
  94
  95 void ejsLexCloseScript(Ejs *ep)
  96 {
  97         EjsInput        *ip;
  98
  99         mprAssert(ep);
 100
 101         ip = ep->input;
 102         mprAssert(ip);
 103         ep->input = ip->next;
 104
 105         mprFree(ip);
 106 }
 107
 108 /******************************************************************************/
 109 /*
 110  *      Initialize an input state structure
 111  */
 112
 113 int ejsInitInputState(EjsInput *ip)
 114 {
 115         mprAssert(ip);
 116
 117         memset(ip, 0, sizeof(*ip));
 118         ip->putBackIndex = -1;
 119
 120         return 0;
 121 }
 122 /******************************************************************************/
 123 /*
 124  *      Save the input state
 125  */
 126
 127 void ejsLexSaveInputState(Ejs *ep, EjsInput *state)
 128 {
 129         EjsInput        *ip;
 130         int                     i;
 131
 132         mprAssert(ep);
 133
 134         ip = ep->input;
 135         mprAssert(ip);
 136
 137         *state = *ip;
 138
 139         for (i = 0; i <= ip->putBackIndex; i++) {
 140                 mprStrcpy(state->putBack[i].tokbuf, EJS_MAX_TOKEN,
 141                         ip->putBack[i].tokbuf);
 142                 state->putBack[i].tid = ip->putBack[i].tid;
 143         }
 144
 145         mprStrcpy(state->line, sizeof(state->line), ip->line);
 146
 147         state->lineColumn = ip->lineColumn;
 148         state->lineNumber = ip->lineNumber;
 149 }
 150
 151 /******************************************************************************/
 152 /*
 153  *      Restore the input state
 154  */
 155
 156 void ejsLexRestoreInputState(Ejs *ep, EjsInput *state)
 157 {
 158         EjsInput        *ip;
 159         EjsToken        *tp;
 160         int                     i;
 161
 162         mprAssert(ep);
 163         mprAssert(state);
 164
 165         ip = ep->input;
 166         mprAssert(ip);
 167
 168         mprStrcpy(ip->tokbuf, sizeof(ip->tokbuf), state->tokbuf);
 169         ip->tokServp = state->tokServp;
 170         ip->tokEndp = state->tokEndp;
 171
 172         ip->script = state->script;
 173         ip->scriptServp = state->scriptServp;
 174         ip->scriptSize = state->scriptSize;
 175
 176         ip->putBackIndex = state->putBackIndex;
 177         for (i = 0; i <= ip->putBackIndex; i++) {
 178                 tp = &ip->putBack[i];
 179                 tp->tid = state->putBack[i].tid;
 180                 mprStrcpy(tp->tokbuf, sizeof(tp->tokbuf), state->putBack[i].tokbuf);
 181         }
 182
 183         mprStrcpy(ip->line, sizeof(ip->line), state->line);
 184
 185         ip->lineColumn = state->lineColumn;
 186         ip->lineNumber = state->lineNumber;
 187 }
 188
 189 /******************************************************************************/
 190 /*
 191  *      Free a saved input state
 192  */
 193
 194 void ejsLexFreeInputState(Ejs *ep, EjsInput *state)
 195 {
 196         mprAssert(ep);
 197         mprAssert(state);
 198
 199         state->putBackIndex = -1;
 200         state->lineColumn = 0;
 201 }
 202
 203 /******************************************************************************/
 204 /*
 205  *      Get the next EJS token
 206  */
 207
 208 int ejsLexGetToken(Ejs *ep, int state)
 209 {
 210         mprAssert(ep);
 211
 212         ep->tid = getLexicalToken(ep, state);
 213         return ep->tid;
 214 }
 215
 216 /******************************************************************************/
 217
 218 /*
 219  *      Check for reserved words "if", "else", "var", "for", "delete", "function",
 220  *      "class", "extends", "public", "private", "protected", "try", "catch",
 221  *      "finally", "throw", "return", "get", "set", "this", "module", "each"
 222  *
 223  *      The "new" and "in" reserved words are handled below. The "true", "false",
 224  *      "null" "typeof" and "undefined" reserved words are handled as global
 225  *      objects.
 226  *
 227  *      Other reserved words not supported:
 228  *              "break", "case", "continue", "default", "do",
 229  *              "instanceof", "switch", "while", "with"
 230  *
 231  *      ECMA extensions reserved words (not supported):
 232  *              "abstract", "boolean", "byte", "char", "const",
 233  *              "debugger", "double", "enum", "export",
 234  *              "final", "float", "goto", "implements", "import", "int",
 235  *              "interface", "long", "native", "package",
 236  *              "short", "static", "super", "synchronized", "transient", "volatile"
 237  *
 238  *      FUTURE -- use a hash lookup
 239  */
 240
 241 static int checkReservedWord(Ejs *ep, int state, int c, int tid)
 242 {
 243         /*      FUTURE -- probably should return for all tokens != EJS_TOK_ID */
 244         /*      FUTURE -- Should have a hash for this. MUCH faster. */
 245
 246         if (!isalpha(ep->token[0]) || tid == EJS_TOK_LITERAL) {
 247                 return tid;
 248         }
 249         if (state == EJS_STATE_STMT) {
 250                 /*      FUTURE OPT -- convert to hash lookup */
 251                 if (strcmp(ep->token, "if") == 0) {
 252                         inputPutback(ep, c);
 253                         return EJS_TOK_IF;
 254                 } else if (strcmp(ep->token, "else") == 0) {
 255                         inputPutback(ep, c);
 256                         return EJS_TOK_ELSE;
 257                 } else if (strcmp(ep->token, "var") == 0) {
 258                         inputPutback(ep, c);
 259                         return EJS_TOK_VAR;
 260                 } else if (strcmp(ep->token, "new") == 0) {
 261                         inputPutback(ep, c);
 262                         return EJS_TOK_NEW;
 263                 } else if (strcmp(ep->token, "for") == 0) {
 264                         inputPutback(ep, c);
 265                         return EJS_TOK_FOR;
 266                 } else if (strcmp(ep->token, "delete") == 0) {
 267                         inputPutback(ep, c);
 268                         return EJS_TOK_DELETE;
 269                 } else if (strcmp(ep->token, "function") == 0) {
 270                         inputPutback(ep, c);
 271                         return EJS_TOK_FUNCTION;
 272                 } else if (strcmp(ep->token, "class") == 0) {
 273                         inputPutback(ep, c);
 274                         return EJS_TOK_CLASS;
 275                 } else if (strcmp(ep->token, "module") == 0) {
 276                         inputPutback(ep, c);
 277                         return EJS_TOK_MODULE;
 278                 } else if (strcmp(ep->token, "extends") == 0) {
 279                         inputPutback(ep, c);
 280                         return EJS_TOK_EXTENDS;
 281                 } else if (strcmp(ep->token, "try") == 0) {
 282                         inputPutback(ep, c);
 283                         return EJS_TOK_TRY;
 284                 } else if (strcmp(ep->token, "catch") == 0) {
 285                         inputPutback(ep, c);
 286                         return EJS_TOK_CATCH;
 287                 } else if (strcmp(ep->token, "finally") == 0) {
 288                         inputPutback(ep, c);
 289                         return EJS_TOK_FINALLY;
 290                 } else if (strcmp(ep->token, "throw") == 0) {
 291                         inputPutback(ep, c);
 292                         return EJS_TOK_THROW;
 293                 } else if (strcmp(ep->token, "public") == 0) {
 294                         inputPutback(ep, c);
 295                         return EJS_TOK_PUBLIC;
 296                 } else if (strcmp(ep->token, "protected") == 0) {
 297                         inputPutback(ep, c);
 298                         return EJS_TOK_PROTECTED;
 299                 } else if (strcmp(ep->token, "private") == 0) {
 300                         inputPutback(ep, c);
 301                         return EJS_TOK_PRIVATE;
 302                 } else if (strcmp(ep->token, "get") == 0) {
 303                         inputPutback(ep, c);
 304                         return EJS_TOK_GET;
 305                 } else if (strcmp(ep->token, "set") == 0) {
 306                         inputPutback(ep, c);
 307                         return EJS_TOK_SET;
 308                 } else if (strcmp(ep->token, "extends") == 0) {
 309                         inputPutback(ep, c);
 310                         return EJS_TOK_EXTENDS;
 311                 } else if (strcmp(ep->token, "try") == 0) {
 312                         inputPutback(ep, c);
 313                         return EJS_TOK_TRY;
 314                 } else if (strcmp(ep->token, "catch") == 0) {
 315                         inputPutback(ep, c);
 316                         return EJS_TOK_CATCH;
 317                 } else if (strcmp(ep->token, "finally") == 0) {
 318                         inputPutback(ep, c);
 319                         return EJS_TOK_FINALLY;
 320                 } else if (strcmp(ep->token, "throw") == 0) {
 321                         inputPutback(ep, c);
 322                         return EJS_TOK_THROW;
 323                 } else if (strcmp(ep->token, "public") == 0) {
 324                         inputPutback(ep, c);
 325                         return EJS_TOK_PUBLIC;
 326                 } else if (strcmp(ep->token, "protected") == 0) {
 327                         inputPutback(ep, c);
 328                         return EJS_TOK_PROTECTED;
 329                 } else if (strcmp(ep->token, "private") == 0) {
 330                         inputPutback(ep, c);
 331                         return EJS_TOK_PRIVATE;
 332                 } else if (strcmp(ep->token, "get") == 0) {
 333                         inputPutback(ep, c);
 334                         return EJS_TOK_GET;
 335                 } else if (strcmp(ep->token, "set") == 0) {
 336                         inputPutback(ep, c);
 337                         return EJS_TOK_SET;
 338                 } else if (strcmp(ep->token, "each") == 0) {
 339                         inputPutback(ep, c);
 340                         return EJS_TOK_EACH;
 341                 } else if (strcmp(ep->token, "return") == 0) {
 342                         if ((c == ';') || (c == '(')) {
 343                                 inputPutback(ep, c);
 344                         }
 345                         return EJS_TOK_RETURN;
 346                 }
 347
 348         } else if (state == EJS_STATE_EXPR) {
 349                 if (strcmp(ep->token, "new") == 0) {
 350                         inputPutback(ep, c);
 351                         return EJS_TOK_NEW;
 352                 } else if (strcmp(ep->token, "in") == 0) {
 353                         inputPutback(ep, c);
 354                         return EJS_TOK_IN;
 355                 } else if (strcmp(ep->token, "function") == 0) {
 356                         inputPutback(ep, c);
 357                         return EJS_TOK_FUNCTION;
 358                 }
 359
 360         } else if (state == EJS_STATE_DEC) {
 361                 if (strcmp(ep->token, "extends") == 0) {
 362                         inputPutback(ep, c);
 363                         return EJS_TOK_EXTENDS;
 364                 }
 365         }
 366         return tid;
 367 }
 368
 369 /******************************************************************************/
 370 /*
 371  *      Get the next EJS token
 372  */
 373
 374 static int getLexicalToken(Ejs *ep, int state)
 375 {
 376         EjsType         type;
 377         EjsInput        *ip;
 378         int                     done, tid, c, quote, style, idx, isHex;
 379
 380         mprAssert(ep);
 381         ip = ep->input;
 382         mprAssert(ip);
 383
 384         ep->tid = -1;
 385         tid = -1;
 386         type = BLD_FEATURE_NUM_TYPE_ID;
 387         isHex = 0;
 388
 389         /*
 390          *      Use a putback tokens first. Don't free strings as caller needs access.
 391          */
 392         if (ip->putBackIndex >= 0) {
 393                 idx = ip->putBackIndex;
 394                 tid = ip->putBack[idx].tid;
 395                 ep->token = (char*) ip->putBack[idx].tokbuf;
 396                 tid = checkReservedWord(ep, state, 0, tid);
 397                 ip->putBackIndex--;
 398                 return tid;
 399         }
 400         ep->token = ip->tokServp = ip->tokEndp = ip->tokbuf;
 401         *ip->tokServp = '\0';
 402
 403         if ((c = inputGetc(ep)) < 0) {
 404                 return EJS_TOK_EOF;
 405         }
 406
 407         /*
 408          *      Main lexical analyser
 409          */
 410         for (done = 0; !done; ) {
 411                 switch (c) {
 412                 case -1:
 413                         return EJS_TOK_EOF;
 414
 415                 case ' ':
 416                 case '\t':
 417                 case '\r':
 418                         do {
 419                                 if ((c = inputGetc(ep)) < 0)
 420                                         break;
 421                         } while (c == ' ' || c == '\t' || c == '\r');
 422                         break;
 423
 424                 case '\n':
 425                         return EJS_TOK_NEWLINE;
 426
 427                 case '(':
 428                         tokenAddChar(ep, c);
 429                         return EJS_TOK_LPAREN;
 430
 431                 case ')':
 432                         tokenAddChar(ep, c);
 433                         return EJS_TOK_RPAREN;
 434
 435                 case '[':
 436                         tokenAddChar(ep, c);
 437                         return EJS_TOK_LBRACKET;
 438
 439                 case ']':
 440                         tokenAddChar(ep, c);
 441                         return EJS_TOK_RBRACKET;
 442
 443                 case '.':
 444                         tokenAddChar(ep, c);
 445                         return EJS_TOK_PERIOD;
 446
 447                 case '{':
 448                         tokenAddChar(ep, c);
 449                         return EJS_TOK_LBRACE;
 450
 451                 case '}':
 452                         tokenAddChar(ep, c);
 453                         return EJS_TOK_RBRACE;
 454
 455                 case '+':
 456                         if ((c = inputGetc(ep)) < 0) {
 457                                 ejsSyntaxError(ep, 0);
 458                                 return EJS_TOK_ERR;
 459                         }
 460                         if (c != '+' ) {
 461                                 inputPutback(ep, c);
 462                                 tokenAddChar(ep, EJS_EXPR_PLUS);
 463                                 return EJS_TOK_EXPR;
 464                         }
 465                         tokenAddChar(ep, EJS_EXPR_INC);
 466                         return EJS_TOK_INC_DEC;
 467
 468                 case '-':
 469                         if ((c = inputGetc(ep)) < 0) {
 470                                 ejsSyntaxError(ep, 0);
 471                                 return EJS_TOK_ERR;
 472                         }
 473                         if (c != '-' ) {
 474                                 inputPutback(ep, c);
 475                                 tokenAddChar(ep, EJS_EXPR_MINUS);
 476                                 return EJS_TOK_EXPR;
 477                         }
 478                         tokenAddChar(ep, EJS_EXPR_DEC);
 479                         return EJS_TOK_INC_DEC;
 480
 481                 case '*':
 482                         tokenAddChar(ep, EJS_EXPR_MUL);
 483                         return EJS_TOK_EXPR;
 484
 485                 case '%':
 486                         tokenAddChar(ep, EJS_EXPR_MOD);
 487                         return EJS_TOK_EXPR;
 488
 489                 case '/':
 490                         /*
 491                          *      Handle the division operator and comments
 492                          */
 493                         if ((c = inputGetc(ep)) < 0) {
 494                                 ejsSyntaxError(ep, 0);
 495                                 return EJS_TOK_ERR;
 496                         }
 497                         if (c != '*' && c != '/') {
 498                                 inputPutback(ep, c);
 499                                 tokenAddChar(ep, EJS_EXPR_DIV);
 500                                 return EJS_TOK_EXPR;
 501                         }
 502                         style = c;
 503                         /*
 504                          *      Eat comments. Both C and C++ comment styles are supported.
 505                          */
 506                         while (1) {
 507                                 if ((c = inputGetc(ep)) < 0) {
 508                                         if (style == '/') {
 509                                                 return EJS_TOK_EOF;
 510                                         }
 511                                         ejsSyntaxError(ep, 0);
 512                                         return EJS_TOK_ERR;
 513                                 }
 514                                 if (c == '\n' && style == '/') {
 515                                         break;
 516                                 } else if (c == '*') {
 517                                         c = inputGetc(ep);
 518                                         if (style == '/') {
 519                                                 if (c == '\n') {
 520                                                         break;
 521                                                 }
 522                                         } else {
 523                                                 if (c == '/') {
 524                                                         break;
 525                                                 }
 526                                         }
 527                                 }
 528                         }
 529                         /*
 530                          *      Continue looking for a token, so get the next character
 531                          */
 532                         if ((c = inputGetc(ep)) < 0) {
 533                                 return EJS_TOK_EOF;
 534                         }
 535                         break;
 536
 537                 case '<':                                                                       /* < and <= */
 538                         if ((c = inputGetc(ep)) < 0) {
 539                                 ejsSyntaxError(ep, 0);
 540                                 return EJS_TOK_ERR;
 541                         }
 542                         if (c == '<') {
 543                                 tokenAddChar(ep, EJS_EXPR_LSHIFT);
 544                                 return EJS_TOK_EXPR;
 545                         } else if (c == '=') {
 546                                 tokenAddChar(ep, EJS_EXPR_LESSEQ);
 547                                 return EJS_TOK_EXPR;
 548                         }
 549                         tokenAddChar(ep, EJS_EXPR_LESS);
 550                         inputPutback(ep, c);
 551                         return EJS_TOK_EXPR;
 552
 553                 case '>':                                                                       /* > and >= */
 554                         if ((c = inputGetc(ep)) < 0) {
 555                                 ejsSyntaxError(ep, 0);
 556                                 return EJS_TOK_ERR;
 557                         }
 558                         if (c == '>') {
 559                                 tokenAddChar(ep, EJS_EXPR_RSHIFT);
 560                                 return EJS_TOK_EXPR;
 561                         } else if (c == '=') {
 562                                 tokenAddChar(ep, EJS_EXPR_GREATEREQ);
 563                                 return EJS_TOK_EXPR;
 564                         }
 565                         tokenAddChar(ep, EJS_EXPR_GREATER);
 566                         inputPutback(ep, c);
 567                         return EJS_TOK_EXPR;
 568
 569                 case '=':                                                                       /* "==" */
 570                         if ((c = inputGetc(ep)) < 0) {
 571                                 ejsSyntaxError(ep, 0);
 572                                 return EJS_TOK_ERR;
 573                         }
 574                         if (c == '=') {
 575                                 tokenAddChar(ep, EJS_EXPR_EQ);
 576                                 return EJS_TOK_EXPR;
 577                         }
 578                         inputPutback(ep, c);
 579                         return EJS_TOK_ASSIGNMENT;
 580
 581                 case '!':                                                                       /* "!=" or "!"*/
 582                         if ((c = inputGetc(ep)) < 0) {
 583                                 ejsSyntaxError(ep, 0);
 584                                 return EJS_TOK_ERR;
 585                         }
 586                         if (c == '=') {
 587                                 tokenAddChar(ep, EJS_EXPR_NOTEQ);
 588                                 return EJS_TOK_EXPR;
 589                         }
 590                         inputPutback(ep, c);
 591                         tokenAddChar(ep, EJS_EXPR_BOOL_COMP);
 592                         return EJS_TOK_EXPR;
 593
 594                 case ';':
 595                         tokenAddChar(ep, c);
 596                         return EJS_TOK_SEMI;
 597
 598                 case ',':
 599                         tokenAddChar(ep, c);
 600                         return EJS_TOK_COMMA;
 601
 602                 case ':':
 603                         tokenAddChar(ep, c);
 604                         return EJS_TOK_COLON;
 605
 606                 case '|':                                                                       /* "||" */
 607                         if ((c = inputGetc(ep)) < 0 || c != '|') {
 608                                 ejsSyntaxError(ep, 0);
 609                                 return EJS_TOK_ERR;
 610                         }
 611                         tokenAddChar(ep, EJS_COND_OR);
 612                         return EJS_TOK_LOGICAL;
 613
 614                 case '&':                                                                       /* "&&" */
 615                         if ((c = inputGetc(ep)) < 0 || c != '&') {
 616                                 ejsSyntaxError(ep, 0);
 617                                 return EJS_TOK_ERR;
 618                         }
 619                         tokenAddChar(ep, EJS_COND_AND);
 620                         return EJS_TOK_LOGICAL;
 621
 622                 case '\"':                                                                      /* String quote */
 623                 case '\'':
 624                         quote = c;
 625                         if ((c = inputGetc(ep)) < 0) {
 626                                 ejsSyntaxError(ep, 0);
 627                                 return EJS_TOK_ERR;
 628                         }
 629
 630                         while (c != quote) {
 631                                 /*
 632                                  *      Check for escape sequence characters
 633                                  */
 634                                 if (c == '\\') {
 635                                         c = inputGetc(ep);
 636
 637                                         if (isdigit(c)) {
 638                                                 /*
 639                                                  *      Octal support, \101 maps to 65 = 'A'. Put first
 640                                                  *      char back so converter will work properly.
 641                                                  */
 642                                                 inputPutback(ep, c);
 643                                                 c = charConvert(ep, 8, 3);
 644
 645                                         } else {
 646                                                 switch (c) {
 647                                                 case 'n':
 648                                                         c = '\n'; break;
 649                                                 case 'b':
 650                                                         c = '\b'; break;
 651                                                 case 'f':
 652                                                         c = '\f'; break;
 653                                                 case 'r':
 654                                                         c = '\r'; break;
 655                                                 case 't':
 656                                                         c = '\t'; break;
 657                                                 case 'x':
 658                                                         /*
 659                                                          *      Hex support, \x41 maps to 65 = 'A'
 660                                                          */
 661                                                         c = charConvert(ep, 16, 2);
 662                                                         break;
 663                                                 case 'u':
 664                                                         /*
 665                                                          *      Unicode support, \x0401 maps to 65 = 'A'
 666                                                          */
 667                                                         c = charConvert(ep, 16, 2);
 668                                                         c = c*16 + charConvert(ep, 16, 2);
 669
 670                                                         break;
 671                                                 case '\'':
 672                                                 case '\"':
 673                                                 case '\\':
 674                                                         break;
 675                                                 default:
 676                                                         if (tokenAddChar(ep, '\\') < 0) {
 677                                                                 return EJS_TOK_ERR;
 678                                                         }
 679                                                 }
 680                                         }
 681                                         if (tokenAddChar(ep, c) < 0) {
 682                                                 return EJS_TOK_ERR;
 683                                         }
 684                                 } else {
 685                                         if (tokenAddChar(ep, c) < 0) {
 686                                                 return EJS_TOK_ERR;
 687                                         }
 688                                 }
 689                                 if ((c = inputGetc(ep)) < 0) {
 690                                         ejsSyntaxError(ep, "Unmatched Quote");
 691                                         return EJS_TOK_ERR;
 692                                 }
 693                         }
 694                         return EJS_TOK_LITERAL;
 695
 696                 case '0':
 697                         if (tokenAddChar(ep, c) < 0) {
 698                                 return EJS_TOK_ERR;
 699                         }
 700                         if ((c = inputGetc(ep)) < 0) {
 701                                 break;
 702                         }
 703                         if (tolower(c) == 'x') {
 704                                 if (tokenAddChar(ep, c) < 0) {
 705                                         return EJS_TOK_ERR;
 706                                 }
 707                                 if ((c = inputGetc(ep)) < 0) {
 708                                         break;
 709                                 }
 710                                 isHex = 1;
 711                                 if (! isxdigit(c)) {
 712                                         parseNumber(ep, type);
 713                                         inputPutback(ep, c);
 714                                         return EJS_TOK_NUMBER;
 715                                 }
 716                         } else if (! isdigit(c)) {
 717 #if BLD_FEATURE_FLOATING_POINT
 718                                 if (c == '.' || tolower(c) == 'e' || c == '+' || c == '-') {
 719                                         /* Fall through */
 720                                         type = EJS_TYPE_FLOAT;
 721                                 } else
 722 #endif
 723                                 {
 724                                         parseNumber(ep, type);
 725                                         inputPutback(ep, c);
 726                                         return EJS_TOK_NUMBER;
 727                                 }
 728                         }
 729                         /* Fall through to get more digits */
 730
 731                 case '1': case '2': case '3': case '4':
 732                 case '5': case '6': case '7': case '8': case '9':
 733                         if (isHex) {
 734                                 do {
 735                                         if (tokenAddChar(ep, c) < 0) {
 736                                                 return EJS_TOK_ERR;
 737                                         }
 738                                         if ((c = inputGetc(ep)) < 0) {
 739                                                 break;
 740                                         }
 741                                 } while (isxdigit(c));
 742
 743                         } else {
 744 #if BLD_FEATURE_FLOATING_POINT
 745                                 do {
 746                                         if (tokenAddChar(ep, c) < 0) {
 747                                                 return EJS_TOK_ERR;
 748                                         }
 749                                         if ((c = inputGetc(ep)) < 0) {
 750                                                 break;
 751                                         }
 752                                         c = tolower(c);
 753                                         if (c == '.' || c == 'e' || c == 'f') {
 754                                                 type = EJS_TYPE_FLOAT;
 755                                         }
 756                                 } while (isdigit(c) || c == '.' || c == 'e' ||
 757                                                 c == 'f' ||
 758                                         ((type == EJS_TYPE_FLOAT) && (c == '+' || c == '-')));
 759 #else
 760                                 do {
 761                                         if (tokenAddChar(ep, c) < 0) {
 762                                                 return EJS_TOK_ERR;
 763                                         }
 764                                         if ((c = inputGetc(ep)) < 0) {
 765                                                 break;
 766                                         }
 767                                 } while (isdigit(c));
 768 #endif
 769                         }
 770
 771                         parseNumber(ep, type);
 772                         inputPutback(ep, c);
 773                         return EJS_TOK_NUMBER;
 774
 775                 default:
 776                         /*
 777                          *      Identifiers or a function names
 778                          */
 779                         while (1) {
 780                                 if (c == '\\') {
 781                                         if ((c = inputGetc(ep)) < 0) {
 782                                                 break;
 783                                         }
 784                                         if (c == '\n' || c == '\r') {
 785                                                 break;
 786                                         }
 787                                 } else if (tokenAddChar(ep, c) < 0) {
 788                                                 break;
 789                                 }
 790                                 if ((c = inputGetc(ep)) < 0) {
 791                                         break;
 792                                 }
 793                                 if (!isalnum(c) && c != '$' && c != '_' &&
 794                                                 c != '\\' && c != '@') {
 795                                         break;
 796                                 }
 797                         }
 798                         if (*ep->token == '\0') {
 799                                 c = inputGetc(ep);
 800                                 break;
 801                         }
 802
 803                         if (! isalpha((int) *ep->token) && *ep->token != '$' &&
 804                                         *ep->token != '_' && *ep->token != '@') {
 805                                 ejsError(ep, EJS_SYNTAX_ERROR, "Invalid identifier %s",
 806                                         ep->token);
 807                                 return EJS_TOK_ERR;
 808                         }
 809
 810                         tid = checkReservedWord(ep, state, c, EJS_TOK_ID);
 811                         if (tid != EJS_TOK_ID) {
 812                                 return tid;
 813                         }
 814
 815                         /*
 816                          *      Skip white space after token to find out whether this is
 817                          *      a function or not.
 818                          */
 819                         while (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
 820                                 if ((c = inputGetc(ep)) < 0)
 821                                         break;
 822                         }
 823
 824                         tid = EJS_TOK_ID;
 825                         if ((strlen(ep->token) + 1) >= EJS_MAX_ID) {
 826                                 ejsError(ep, EJS_SYNTAX_ERROR,
 827                                         "Identifier too big. Max is %d letters.", EJS_MAX_ID);
 828                                 return EJS_TOK_ERR;
 829                         }
 830                         done++;
 831                 }
 832         }
 833
 834         /*
 835          *      Putback the last extra character for next time
 836          */
 837         inputPutback(ep, c);
 838         return tid;
 839 }
 840
 841 /******************************************************************************/
 842
 843 static void parseNumber(Ejs *ep, EjsType type)
 844 {
 845         switch (type) {
 846         case EJS_TYPE_INT:
 847                 ep->tokenNumber.integer = ejsParseInteger(ep->token);
 848                 ep->tokenNumber.type = type;
 849                 break;
 850
 851 #if BLD_FEATURE_FLOATING_POINT
 852         case EJS_TYPE_FLOAT:
 853                 ep->tokenNumber.floating = atof(ep->token);
 854                 ep->tokenNumber.type = type;
 855                 break;
 856 #endif
 857
 858 #if BLD_FEATURE_INT64
 859         case EJS_TYPE_INT64:
 860                 ep->tokenNumber.integer64 = ejsParseInteger64(ep->token);
 861                 ep->tokenNumber.type = type;
 862                 break;
 863 #endif
 864         }
 865 }
 866
 867 /******************************************************************************/
 868 /*
 869  *      Convert a hex or octal character back to binary, return original char if
 870  *      not a hex digit
 871  */
 872
 873 static int charConvert(Ejs *ep, int base, int maxDig)
 874 {
 875         int             i, c, lval, convChar;
 876
 877         lval = 0;
 878         for (i = 0; i < maxDig; i++) {
 879                 if ((c = inputGetc(ep)) < 0) {
 880                         break;
 881                 }
 882                 /*
 883                  *      Initialize to out of range value
 884                  */
 885                 convChar = base;
 886                 if (isdigit(c)) {
 887                         convChar = c - '0';
 888                 } else if (c >= 'a' && c <= 'f') {
 889                         convChar = c - 'a' + 10;
 890                 } else if (c >= 'A' && c <= 'F') {
 891                         convChar = c - 'A' + 10;
 892                 }
 893                 /*
 894                  *      If unexpected character then return it to buffer.
 895                  */
 896                 if (convChar >= base) {
 897                         inputPutback(ep, c);
 898                         break;
 899                 }
 900                 lval = (lval * base) + convChar;
 901         }
 902         return lval;
 903 }
 904
 905 /******************************************************************************/
 906 /*
 907  *      Putback the last token read. Accept at most one push back token.
 908  */
 909
 910 void ejsLexPutbackToken(Ejs *ep, int tid, char *string)
 911 {
 912         EjsInput        *ip;
 913         EjsToken        *tp;
 914         int                     idx;
 915
 916         mprAssert(ep);
 917         ip = ep->input;
 918         mprAssert(ip);
 919
 920         ip->putBackIndex += 1;
 921
 922         mprAssert(ip->putBackIndex < EJS_TOKEN_STACK);
 923         idx = ip->putBackIndex;
 924
 925         tp = &ip->putBack[idx];
 926         tp->tid = tid;
 927
 928         mprStrcpy(tp->tokbuf, sizeof(tp->tokbuf), string);
 929 }
 930
 931 /******************************************************************************/
 932 /*
 933  *      Add a character to the token buffer
 934  */
 935
 936 static int tokenAddChar(Ejs *ep, int c)
 937 {
 938         EjsInput        *ip;
 939
 940         mprAssert(ep);
 941         ip = ep->input;
 942         mprAssert(ip);
 943
 944         if (ip->tokEndp >= &ip->tokbuf[sizeof(ip->tokbuf) - 1]) {
 945                 ejsSyntaxError(ep, "Token too big");
 946                 return -1;
 947         }
 948         *ip->tokEndp++ = c;
 949         *ip->tokEndp = '\0';
 950
 951         return 0;
 952 }
 953
 954 /******************************************************************************/
 955 /*
 956  *      Get another input character
 957  */
 958
 959 static int inputGetc(Ejs *ep)
 960 {
 961         EjsInput        *ip;
 962         int                     c;
 963
 964         mprAssert(ep);
 965         ip = ep->input;
 966
 967         if (ip->scriptSize <= 0) {
 968                 return -1;
 969         }
 970
 971         c = (uchar) (*ip->scriptServp++);
 972         ip->scriptSize--;
 973
 974         /*
 975          *      For debugging, accumulate the line number and the currenly parsed line
 976          */
 977         if (c == '\n') {
 978 #if 0 && BLD_DEBUG
 979                 if (ip->lineColumn > 0) {
 980                         printf("PARSED: %s\n", ip->line);
 981                 }
 982 #endif
 983                 ip->lineNumber++;
 984                 ip->lineColumn = 0;
 985         } else if ((ip->lineColumn + 2) < sizeof(ip->line)) {
 986                 ip->line[ip->lineColumn++] = c;
 987                 ip->line[ip->lineColumn] = '\0';
 988         }
 989         return c;
 990 }
 991
 992 /******************************************************************************/
 993 /*
 994  *      Putback a character onto the input queue
 995  */
 996
 997 static void inputPutback(Ejs *ep, int c)
 998 {
 999         EjsInput        *ip;
1000
1001         mprAssert(ep);
1002
1003         if (c > 0) {
1004                 ip = ep->input;
1005                 *--ip->scriptServp = c;
1006                 ip->scriptSize++;
1007                 if (--(ip->lineColumn) < 0) {
1008                         ip->lineColumn = 0;
1009                 }
1010                 mprAssert(ip->line);
1011                 mprAssert(ip->lineColumn >= 0);
1012                 mprAssert(ip->lineColumn < sizeof(ip->line));
1013                 ip->line[ip->lineColumn] = '\0';
1014         }
1015 }
1016
1017 /******************************************************************************/
1018
1019 #else
1020 void ejsLexDummy() {}
1021
1022 /******************************************************************************/
1023 #endif /* BLD_FEATURE_EJS */
1024
1025 /*
1026  * Local variables:
1027  * tab-width: 4
1028  * c-basic-offset: 4
1029  * End:
1030  * vim:tw=78
1031  * vim600: sw=4 ts=4 fdm=marker
1032  * vim<600: sw=4 ts=4
1033  */