source4/lib/appweb/ejs/ejsLex.c

   1 /*
   2  *      @file   ejsLex.c
   3  *      @brief  EJS Lexical Analyser
   4  *      @overview EJS lexical analyser. This implementes a lexical analyser
   5  *              for a subset of the JavaScript language.
   6  */
   7 /********************************* Copyright **********************************/
   8 /*
   9  *      @copy   default.g
  10  *
  11  *      Copyright (c) Mbedthis Software LLC, 2003-2005. All Rights Reserved.
  12  *      Portions Copyright (c) GoAhead Software, 1995-2000. All Rights Reserved.
  13  *
  14  *      This software is distributed under commercial and open source licenses.
  15  *      You may use the GPL open source license described below or you may acquire
  16  *      a commercial license from Mbedthis Software. You agree to be fully bound
  17  *      by the terms of either license. Consult the LICENSE.TXT distributed with
  18  *      this software for full details.
  19  *
  20  *      This software is open source; you can redistribute it and/or modify it
  21  *      under the terms of the GNU General Public License as published by the
  22  *      Free Software Foundation; either version 2 of the License, or (at your
  23  *      option) any later version. See the GNU General Public License for more
  24  *      details at: http://www.mbedthis.com/downloads/gplLicense.html
  25  *
  26  *      This program is distributed WITHOUT ANY WARRANTY; without even the
  27  *      implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  28  *
  29  *      This GPL license does NOT permit incorporating this software into
  30  *      proprietary programs. If you are unable to comply with the GPL, you must
  31  *      acquire a commercial license to use this software. Commercial licenses
  32  *      for this software and support services are available from Mbedthis
  33  *      Software at http://www.mbedthis.com
  34  *
  35  *      @end
  36  */
  37 /********************************** Includes **********************************/
  38
  39 #include        "ejsInternal.h"
  40
  41 #if BLD_FEATURE_EJS
  42
  43 /****************************** Forward Declarations **************************/
  44
  45 static int              getLexicalToken(Ejs *ep, int state);
  46 static int              tokenAddChar(Ejs *ep, int c);
  47 static int              inputGetc(Ejs *ep);
  48 static void             inputPutback(Ejs *ep, int c);
  49 static int              charConvert(Ejs *ep, int base, int maxDig);
  50
  51 /************************************* Code ***********************************/
  52 /*
  53  *      Open a new input script
  54  */
  55
  56 int ejsLexOpenScript(Ejs *ep, char *script)
  57 {
  58         EjsInput        *ip;
  59
  60         mprAssert(ep);
  61         mprAssert(script);
  62
  63         if ((ip = mprMalloc(sizeof(EjsInput))) == NULL) {
  64                 return -1;
  65         }
  66         memset(ip, 0, sizeof(*ip));
  67         ip->next = ep->input;
  68         ep->input = ip;
  69         ip->procName = ep->proc?ep->proc->procName:NULL;
  70
  71 /*
  72  *      Create the parse token buffer and script buffer
  73  */
  74         ip->tokbuf = mprMalloc(EJS_PARSE_INCR);
  75         ip->tokSize = EJS_PARSE_INCR;
  76         ip->tokServp = ip->tokbuf;
  77         ip->tokEndp = ip->tokbuf;
  78
  79         ip->script = mprStrdup(script);
  80         ip->scriptSize = strlen(script);
  81         ip->scriptServp = ip->script;
  82
  83         ip->lineNumber = 1;
  84         ip->lineLength = 0;
  85         ip->lineColumn = 0;
  86         ip->line = NULL;
  87
  88         ip->putBackIndex = -1;
  89
  90         return 0;
  91 }
  92
  93 /******************************************************************************/
  94 /*
  95  *      Close the input script
  96  */
  97
  98 void ejsLexCloseScript(Ejs *ep)
  99 {
 100         EjsInput        *ip;
 101         int                     i;
 102
 103         mprAssert(ep);
 104
 105         ip = ep->input;
 106         mprAssert(ip);
 107         ep->input = ip->next;
 108
 109         for (i = 0; i < EJS_TOKEN_STACK; i++) {
 110                 mprFree(ip->putBack[i].token);
 111                 ip->putBack[i].token = 0;
 112         }
 113
 114         mprFree(ip->line);
 115         mprFree(ip->tokbuf);
 116         mprFree(ip->script);
 117
 118         mprFree(ip);
 119 }
 120
 121 /******************************************************************************/
 122 /*
 123  *      Initialize an input state structure
 124  */
 125
 126 int ejsInitInputState(EjsInput *ip)
 127 {
 128         mprAssert(ip);
 129
 130         memset(ip, 0, sizeof(*ip));
 131         ip->putBackIndex = -1;
 132
 133         return 0;
 134 }
 135 /******************************************************************************/
 136 /*
 137  *      Save the input state
 138  */
 139
 140 void ejsLexSaveInputState(Ejs *ep, EjsInput *state)
 141 {
 142         EjsInput        *ip;
 143         int                     i;
 144
 145         mprAssert(ep);
 146
 147         ip = ep->input;
 148         mprAssert(ip);
 149
 150         *state = *ip;
 151
 152         for (i = 0; i < ip->putBackIndex; i++) {
 153                 state->putBack[i].token = mprStrdup(ip->putBack[i].token);
 154                 state->putBack[i].id = ip->putBack[i].id;
 155         }
 156         for (; i < EJS_TOKEN_STACK; i++) {
 157                 state->putBack[i].token = 0;
 158         }
 159
 160         state->line = mprMalloc(ip->lineLength);
 161         mprStrcpy(state->line, ip->lineLength, ip->line);
 162
 163         state->lineColumn = ip->lineColumn;
 164         state->lineNumber = ip->lineNumber;
 165         state->lineLength = ip->lineLength;
 166 }
 167
 168 /******************************************************************************/
 169 /*
 170  *      Restore the input state
 171  */
 172
 173 void ejsLexRestoreInputState(Ejs *ep, EjsInput *state)
 174 {
 175         EjsInput        *ip;
 176         int                     i;
 177
 178         mprAssert(ep);
 179         mprAssert(state);
 180
 181         ip = ep->input;
 182         mprAssert(ip);
 183
 184         ip->tokbuf = state->tokbuf;
 185         ip->tokServp = state->tokServp;
 186         ip->tokEndp = state->tokEndp;
 187         ip->tokSize = state->tokSize;
 188
 189         ip->script = state->script;
 190         ip->scriptServp = state->scriptServp;
 191         ip->scriptSize = state->scriptSize;
 192
 193         ip->putBackIndex = state->putBackIndex;
 194         for (i = 0; i < ip->putBackIndex; i++) {
 195                 mprFree(ip->putBack[i].token);
 196                 ip->putBack[i].id = state->putBack[i].id;
 197                 ip->putBack[i].token = mprStrdup(state->putBack[i].token);
 198         }
 199
 200         mprFree(ip->line);
 201         ip->line = mprMalloc(state->lineLength);
 202         mprStrcpy(ip->line, state->lineLength, state->line);
 203
 204         ip->lineColumn = state->lineColumn;
 205         ip->lineNumber = state->lineNumber;
 206         ip->lineLength = state->lineLength;
 207 }
 208
 209 /******************************************************************************/
 210 /*
 211  *      Free a saved input state
 212  */
 213
 214 void ejsLexFreeInputState(Ejs *ep, EjsInput *state)
 215 {
 216         int                     i;
 217
 218         mprAssert(ep);
 219         mprAssert(state);
 220
 221         for (i = 0; i < EJS_TOKEN_STACK; i++) {
 222                 mprFree(state->putBack[i].token);
 223         }
 224         state->putBackIndex = -1;
 225         mprFree(state->line);
 226         state->lineLength = 0;
 227         state->lineColumn = 0;
 228 }
 229
 230 /******************************************************************************/
 231 /*
 232  *      Get the next EJS token
 233  */
 234
 235 int ejsLexGetToken(Ejs *ep, int state)
 236 {
 237         mprAssert(ep);
 238
 239         ep->tid = getLexicalToken(ep, state);
 240         return ep->tid;
 241 }
 242
 243 /******************************************************************************/
 244
 245 /*
 246  *      Check for reserved words "if", "else", "var", "for", "foreach",
 247  *      "delete", "function", and "return". "new", "in" and "function"
 248  *      done below. "true", "false", "null", "undefined" are handled
 249  *      as global objects.
 250  *
 251  *      Other reserved words not supported:
 252  *              "break", "case", "catch", "continue", "default", "do",
 253  *              "finally", "instanceof", "switch", "this", "throw", "try",
 254  *              "typeof", "while", "with"
 255  *
 256  *      ECMA extensions reserved words (not supported):
 257  *              "abstract", "boolean", "byte", "char", "class", "const",
 258  *              "debugger", "double", "enum", "export", "extends",
 259  *              "final", "float", "goto", "implements", "import", "int",
 260  *              "interface", "long", "native", "package", "private",
 261  *              "protected", "public", "short", "static", "super",
 262  *              "synchronized", "throws", "transient", "volatile"
 263  */
 264
 265 static int checkReservedWord(Ejs *ep, int state, int c, int tid)
 266 {
 267         if (state == EJS_STATE_STMT) {
 268                 if (strcmp(ep->token, "if") == 0) {
 269                         inputPutback(ep, c);
 270                         return EJS_TOK_IF;
 271                 } else if (strcmp(ep->token, "else") == 0) {
 272                         inputPutback(ep, c);
 273                         return EJS_TOK_ELSE;
 274                 } else if (strcmp(ep->token, "var") == 0) {
 275                         inputPutback(ep, c);
 276                         return EJS_TOK_VAR;
 277                 } else if (strcmp(ep->token, "for") == 0) {
 278                         inputPutback(ep, c);
 279                         return EJS_TOK_FOR;
 280                 } else if (strcmp(ep->token, "delete") == 0) {
 281                         inputPutback(ep, c);
 282                         return EJS_TOK_DELETE;
 283                 } else if (strcmp(ep->token, "function") == 0) {
 284                         inputPutback(ep, c);
 285                         return EJS_TOK_FUNCTION;
 286                 } else if (strcmp(ep->token, "return") == 0) {
 287                         if ((c == ';') || (c == '(')) {
 288                                 inputPutback(ep, c);
 289                         }
 290                         return EJS_TOK_RETURN;
 291                 }
 292         } else if (state == EJS_STATE_EXPR) {
 293                 if (strcmp(ep->token, "new") == 0) {
 294                         inputPutback(ep, c);
 295                         return EJS_TOK_NEW;
 296                 } else if (strcmp(ep->token, "in") == 0) {
 297                         inputPutback(ep, c);
 298                         return EJS_TOK_IN;
 299                 } else if (strcmp(ep->token, "function") == 0) {
 300                         inputPutback(ep, c);
 301                         return EJS_TOK_FUNCTION;
 302                 }
 303         }
 304         return tid;
 305 }
 306
 307 /******************************************************************************/
 308 /*
 309  *      Get the next EJS token
 310  */
 311
 312 static int getLexicalToken(Ejs *ep, int state)
 313 {
 314         MprType         type;
 315         EjsInput        *ip;
 316         int                     done, tid, c, quote, style, idx;
 317
 318         mprAssert(ep);
 319         ip = ep->input;
 320         mprAssert(ip);
 321
 322         ep->tid = -1;
 323         tid = -1;
 324         type = BLD_FEATURE_NUM_TYPE_ID;
 325
 326         /*
 327          *      Use a putback tokens first. Don't free strings as caller needs access.
 328          */
 329         if (ip->putBackIndex >= 0) {
 330                 idx = ip->putBackIndex;
 331                 tid = ip->putBack[idx].id;
 332                 ep->token = (char*) ip->putBack[idx].token;
 333                 tid = checkReservedWord(ep, state, 0, tid);
 334                 ip->putBackIndex--;
 335                 return tid;
 336         }
 337         ep->token = ip->tokServp = ip->tokEndp = ip->tokbuf;
 338         *ip->tokServp = '\0';
 339
 340         if ((c = inputGetc(ep)) < 0) {
 341                 return EJS_TOK_EOF;
 342         }
 343
 344         /*
 345          *      Main lexical analyser
 346          */
 347         for (done = 0; !done; ) {
 348                 switch (c) {
 349                 case -1:
 350                         return EJS_TOK_EOF;
 351
 352                 case ' ':
 353                 case '\t':
 354                 case '\r':
 355                         do {
 356                                 if ((c = inputGetc(ep)) < 0)
 357                                         break;
 358                         } while (c == ' ' || c == '\t' || c == '\r');
 359                         break;
 360
 361                 case '\n':
 362                         return EJS_TOK_NEWLINE;
 363
 364                 case '(':
 365                         tokenAddChar(ep, c);
 366                         return EJS_TOK_LPAREN;
 367
 368                 case ')':
 369                         tokenAddChar(ep, c);
 370                         return EJS_TOK_RPAREN;
 371
 372                 case '[':
 373                         tokenAddChar(ep, c);
 374                         return EJS_TOK_LBRACKET;
 375
 376                 case ']':
 377                         tokenAddChar(ep, c);
 378                         return EJS_TOK_RBRACKET;
 379
 380                 case '.':
 381                         tokenAddChar(ep, c);
 382                         return EJS_TOK_PERIOD;
 383
 384                 case '{':
 385                         tokenAddChar(ep, c);
 386                         return EJS_TOK_LBRACE;
 387
 388                 case '}':
 389                         tokenAddChar(ep, c);
 390                         return EJS_TOK_RBRACE;
 391
 392                 case '+':
 393                         if ((c = inputGetc(ep)) < 0) {
 394                                 ejsError(ep, "Syntax Error");
 395                                 return EJS_TOK_ERR;
 396                         }
 397                         if (c != '+' ) {
 398                                 inputPutback(ep, c);
 399                                 tokenAddChar(ep, EJS_EXPR_PLUS);
 400                                 return EJS_TOK_EXPR;
 401                         }
 402                         tokenAddChar(ep, EJS_EXPR_INC);
 403                         return EJS_TOK_INC_DEC;
 404
 405                 case '-':
 406                         if ((c = inputGetc(ep)) < 0) {
 407                                 ejsError(ep, "Syntax Error");
 408                                 return EJS_TOK_ERR;
 409                         }
 410                         if (c != '-' ) {
 411                                 inputPutback(ep, c);
 412                                 tokenAddChar(ep, EJS_EXPR_MINUS);
 413                                 return EJS_TOK_EXPR;
 414                         }
 415                         tokenAddChar(ep, EJS_EXPR_DEC);
 416                         return EJS_TOK_INC_DEC;
 417
 418                 case '*':
 419                         tokenAddChar(ep, EJS_EXPR_MUL);
 420                         return EJS_TOK_EXPR;
 421
 422                 case '%':
 423                         tokenAddChar(ep, EJS_EXPR_MOD);
 424                         return EJS_TOK_EXPR;
 425
 426                 case '/':
 427                         /*
 428                          *      Handle the division operator and comments
 429                          */
 430                         if ((c = inputGetc(ep)) < 0) {
 431                                 ejsError(ep, "Syntax Error");
 432                                 return EJS_TOK_ERR;
 433                         }
 434                         if (c != '*' && c != '/') {
 435                                 inputPutback(ep, c);
 436                                 tokenAddChar(ep, EJS_EXPR_DIV);
 437                                 return EJS_TOK_EXPR;
 438                         }
 439                         style = c;
 440                         /*
 441                          *      Eat comments. Both C and C++ comment styles are supported.
 442                          */
 443                         while (1) {
 444                                 if ((c = inputGetc(ep)) < 0) {
 445                                         if (style == '/') {
 446                                                 return EJS_TOK_EOF;
 447                                         }
 448                                         ejsError(ep, "Syntax Error");
 449                                         return EJS_TOK_ERR;
 450                                 }
 451                                 if (c == '\n' && style == '/') {
 452                                         break;
 453                                 } else if (c == '*') {
 454                                         c = inputGetc(ep);
 455                                         if (style == '/') {
 456                                                 if (c == '\n') {
 457                                                         break;
 458                                                 }
 459                                         } else {
 460                                                 if (c == '/') {
 461                                                         break;
 462                                                 }
 463                                         }
 464                                 }
 465                         }
 466                         /*
 467                          *      Continue looking for a token, so get the next character
 468                          */
 469                         if ((c = inputGetc(ep)) < 0) {
 470                                 return EJS_TOK_EOF;
 471                         }
 472                         break;
 473
 474                 case '<':                                                                       /* < and <= */
 475                         if ((c = inputGetc(ep)) < 0) {
 476                                 ejsError(ep, "Syntax Error");
 477                                 return EJS_TOK_ERR;
 478                         }
 479                         if (c == '<') {
 480                                 tokenAddChar(ep, EJS_EXPR_LSHIFT);
 481                                 return EJS_TOK_EXPR;
 482                         } else if (c == '=') {
 483                                 tokenAddChar(ep, EJS_EXPR_LESSEQ);
 484                                 return EJS_TOK_EXPR;
 485                         }
 486                         tokenAddChar(ep, EJS_EXPR_LESS);
 487                         inputPutback(ep, c);
 488                         return EJS_TOK_EXPR;
 489
 490                 case '>':                                                                       /* > and >= */
 491                         if ((c = inputGetc(ep)) < 0) {
 492                                 ejsError(ep, "Syntax Error");
 493                                 return EJS_TOK_ERR;
 494                         }
 495                         if (c == '>') {
 496                                 tokenAddChar(ep, EJS_EXPR_RSHIFT);
 497                                 return EJS_TOK_EXPR;
 498                         } else if (c == '=') {
 499                                 tokenAddChar(ep, EJS_EXPR_GREATEREQ);
 500                                 return EJS_TOK_EXPR;
 501                         }
 502                         tokenAddChar(ep, EJS_EXPR_GREATER);
 503                         inputPutback(ep, c);
 504                         return EJS_TOK_EXPR;
 505
 506                 case '=':                                                                       /* "==" */
 507                         if ((c = inputGetc(ep)) < 0) {
 508                                 ejsError(ep, "Syntax Error");
 509                                 return EJS_TOK_ERR;
 510                         }
 511                         if (c == '=') {
 512                                 tokenAddChar(ep, EJS_EXPR_EQ);
 513                                 return EJS_TOK_EXPR;
 514                         }
 515                         inputPutback(ep, c);
 516                         return EJS_TOK_ASSIGNMENT;
 517
 518                 case '!':                                                                       /* "!=" or "!"*/
 519                         if ((c = inputGetc(ep)) < 0) {
 520                                 ejsError(ep, "Syntax Error");
 521                                 return EJS_TOK_ERR;
 522                         }
 523                         if (c == '=') {
 524                                 tokenAddChar(ep, EJS_EXPR_NOTEQ);
 525                                 return EJS_TOK_EXPR;
 526                         }
 527                         inputPutback(ep, c);
 528                         tokenAddChar(ep, EJS_EXPR_BOOL_COMP);
 529                         return EJS_TOK_EXPR;
 530
 531                 case ';':
 532                         tokenAddChar(ep, c);
 533                         return EJS_TOK_SEMI;
 534
 535                 case ',':
 536                         tokenAddChar(ep, c);
 537                         return EJS_TOK_COMMA;
 538
 539                 case '|':                                                                       /* "||" */
 540                         if ((c = inputGetc(ep)) < 0 || c != '|') {
 541                                 ejsError(ep, "Syntax Error");
 542                                 return EJS_TOK_ERR;
 543                         }
 544                         tokenAddChar(ep, EJS_COND_OR);
 545                         return EJS_TOK_LOGICAL;
 546
 547                 case '&':                                                                       /* "&&" */
 548                         if ((c = inputGetc(ep)) < 0 || c != '&') {
 549                                 ejsError(ep, "Syntax Error");
 550                                 return EJS_TOK_ERR;
 551                         }
 552                         tokenAddChar(ep, EJS_COND_AND);
 553                         return EJS_TOK_LOGICAL;
 554
 555                 case '\"':                                                                      /* String quote */
 556                 case '\'':
 557                         quote = c;
 558                         if ((c = inputGetc(ep)) < 0) {
 559                                 ejsError(ep, "Syntax Error");
 560                                 return EJS_TOK_ERR;
 561                         }
 562
 563                         while (c != quote) {
 564                                 /*
 565                                  *      Check for escape sequence characters
 566                                  */
 567                                 if (c == '\\') {
 568                                         c = inputGetc(ep);
 569
 570                                         if (isdigit(c)) {
 571                                                 /*
 572                                                  *      Octal support, \101 maps to 65 = 'A'. Put first
 573                                                  *      char back so converter will work properly.
 574                                                  */
 575                                                 inputPutback(ep, c);
 576                                                 c = charConvert(ep, 8, 3);
 577
 578                                         } else {
 579                                                 switch (c) {
 580                                                 case 'n':
 581                                                         c = '\n'; break;
 582                                                 case 'b':
 583                                                         c = '\b'; break;
 584                                                 case 'f':
 585                                                         c = '\f'; break;
 586                                                 case 'r':
 587                                                         c = '\r'; break;
 588                                                 case 't':
 589                                                         c = '\t'; break;
 590                                                 case 'x':
 591                                                         /*
 592                                                          *      Hex support, \x41 maps to 65 = 'A'
 593                                                          */
 594                                                         c = charConvert(ep, 16, 2);
 595                                                         break;
 596                                                 case 'u':
 597                                                         /*
 598                                                          *      Unicode support, \x0401 maps to 65 = 'A'
 599                                                          */
 600                                                         c = charConvert(ep, 16, 2);
 601                                                         c = c*16 + charConvert(ep, 16, 2);
 602
 603                                                         break;
 604                                                 case '\'':
 605                                                 case '\"':
 606                                                 case '\\':
 607                                                         break;
 608                                                 default:
 609                                                         ejsError(ep, "Invalid Escape Sequence");
 610                                                         return EJS_TOK_ERR;
 611                                                 }
 612                                         }
 613                                         if (tokenAddChar(ep, c) < 0) {
 614                                                 return EJS_TOK_ERR;
 615                                         }
 616                                 } else {
 617                                         if (tokenAddChar(ep, c) < 0) {
 618                                                 return EJS_TOK_ERR;
 619                                         }
 620                                 }
 621                                 if ((c = inputGetc(ep)) < 0) {
 622                                         ejsError(ep, "Unmatched Quote");
 623                                         return EJS_TOK_ERR;
 624                                 }
 625                         }
 626                         return EJS_TOK_LITERAL;
 627
 628                 case '0':
 629                         if (tokenAddChar(ep, c) < 0) {
 630                                 return EJS_TOK_ERR;
 631                         }
 632                         if ((c = inputGetc(ep)) < 0) {
 633                                 break;
 634                         }
 635                         if (tolower(c) == 'x') {
 636                                 do {
 637                                         if (tokenAddChar(ep, c) < 0) {
 638                                                 return EJS_TOK_ERR;
 639                                         }
 640                                         if ((c = inputGetc(ep)) < 0) {
 641                                                 break;
 642                                         }
 643                                 } while (isdigit(c) || (tolower(c) >= 'a' && tolower(c) <= 'f'));
 644
 645                                 mprDestroyVar(&ep->tokenNumber);
 646                                 ep->tokenNumber = mprParseVar(ep->token, type);
 647                                 inputPutback(ep, c);
 648                                 return EJS_TOK_NUMBER;
 649                         }
 650                         if (! isdigit(c)) {
 651 #if BLD_FEATURE_FLOATING_POINT
 652                                 if (c == '.' || tolower(c) == 'e' || c == '+' || c == '-') {
 653                                         /* Fall through */
 654                                         type = MPR_TYPE_FLOAT;
 655                                 } else
 656 #endif
 657                                 {
 658                                         mprDestroyVar(&ep->tokenNumber);
 659                                         ep->tokenNumber = mprParseVar(ep->token, type);
 660                                         inputPutback(ep, c);
 661                                         return EJS_TOK_NUMBER;
 662                                 }
 663                         }
 664                         /* Fall through to get more digits */
 665
 666                 case '1': case '2': case '3': case '4':
 667                 case '5': case '6': case '7': case '8': case '9':
 668                         do {
 669                                 if (tokenAddChar(ep, c) < 0) {
 670                                         return EJS_TOK_ERR;
 671                                 }
 672                                 if ((c = inputGetc(ep)) < 0) {
 673                                         break;
 674                                 }
 675 #if BLD_FEATURE_FLOATING_POINT
 676                                 if (c == '.' || tolower(c) == 'e' || tolower(c) == 'f') {
 677                                         type = MPR_TYPE_FLOAT;
 678                                 }
 679                         } while (isdigit(c) || c == '.' || tolower(c) == 'e' || tolower(c) == 'f' ||
 680                                 ((type == MPR_TYPE_FLOAT) && (c == '+' || c == '-')));
 681 #else
 682                         } while (isdigit(c));
 683 #endif
 684
 685                         mprDestroyVar(&ep->tokenNumber);
 686                         ep->tokenNumber = mprParseVar(ep->token, type);
 687                         inputPutback(ep, c);
 688                         return EJS_TOK_NUMBER;
 689
 690                 default:
 691                         /*
 692                          *      Identifiers or a function names
 693                          */
 694                         while (1) {
 695                                 if (c == '\\') {
 696                                         if ((c = inputGetc(ep)) < 0) {
 697                                                 break;
 698                                         }
 699                                         if (c == '\n' || c == '\r') {
 700                                                 break;
 701                                         }
 702                                 } else if (tokenAddChar(ep, c) < 0) {
 703                                                 break;
 704                                 }
 705                                 if ((c = inputGetc(ep)) < 0) {
 706                                         break;
 707                                 }
 708                                 if (!isalnum(c) && c != '$' && c != '_' && c != '\\') {
 709                                         break;
 710                                 }
 711                         }
 712                         if (*ep->token == '\0') {
 713                                 c = inputGetc(ep);
 714                                 break;
 715                         }
 716                         if (! isalpha((int) *ep->token) && *ep->token != '$' &&
 717                                         *ep->token != '_') {
 718                                 ejsError(ep, "Invalid identifier %s", ep->token);
 719                                 return EJS_TOK_ERR;
 720                         }
 721
 722                         tid = checkReservedWord(ep, state, c, EJS_TOK_ID);
 723                         if (tid != EJS_TOK_ID) {
 724                                 return tid;
 725                         }
 726
 727                         /*
 728                          *      Skip white space after token to find out whether this is
 729                          *      a function or not.
 730                          */
 731                         while (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
 732                                 if ((c = inputGetc(ep)) < 0)
 733                                         break;
 734                         }
 735
 736                         tid = EJS_TOK_ID;
 737                         done++;
 738                 }
 739         }
 740
 741         /*
 742          *      Putback the last extra character for next time
 743          */
 744         inputPutback(ep, c);
 745         return tid;
 746 }
 747
 748 /******************************************************************************/
 749 /*
 750  *      Convert a hex or octal character back to binary, return original char if
 751  *      not a hex digit
 752  */
 753
 754 static int charConvert(Ejs *ep, int base, int maxDig)
 755 {
 756         int             i, c, lval, convChar;
 757
 758         lval = 0;
 759         for (i = 0; i < maxDig; i++) {
 760                 if ((c = inputGetc(ep)) < 0) {
 761                         break;
 762                 }
 763                 /*
 764                  *      Initialize to out of range value
 765                  */
 766                 convChar = base;
 767                 if (isdigit(c)) {
 768                         convChar = c - '0';
 769                 } else if (c >= 'a' && c <= 'f') {
 770                         convChar = c - 'a' + 10;
 771                 } else if (c >= 'A' && c <= 'F') {
 772                         convChar = c - 'A' + 10;
 773                 }
 774                 /*
 775                  *      If unexpected character then return it to buffer.
 776                  */
 777                 if (convChar >= base) {
 778                         inputPutback(ep, c);
 779                         break;
 780                 }
 781                 lval = (lval * base) + convChar;
 782         }
 783         return lval;
 784 }
 785
 786 /******************************************************************************/
 787 /*
 788  *      Putback the last token read. Accept at most one push back token.
 789  */
 790
 791 void ejsLexPutbackToken(Ejs *ep, int tid, char *string)
 792 {
 793         EjsInput        *ip;
 794         int                     idx;
 795
 796         mprAssert(ep);
 797         ip = ep->input;
 798         mprAssert(ip);
 799
 800         ip->putBackIndex += 1;
 801         idx = ip->putBackIndex;
 802         ip->putBack[idx].id = tid;
 803
 804         if (ip->putBack[idx].token) {
 805                 if (ip->putBack[idx].token == string) {
 806                         return;
 807                 }
 808                 mprFree(ip->putBack[idx].token);
 809         }
 810         ip->putBack[idx].token = mprStrdup(string);
 811 }
 812
 813 /******************************************************************************/
 814 /*
 815  *      Add a character to the token buffer
 816  */
 817
 818 static int tokenAddChar(Ejs *ep, int c)
 819 {
 820         EjsInput        *ip;
 821         uchar           *oldbuf;
 822
 823         mprAssert(ep);
 824         ip = ep->input;
 825         mprAssert(ip);
 826
 827         if (ip->tokEndp >= &ip->tokbuf[ip->tokSize - 1]) {
 828                 ip->tokSize += EJS_PARSE_INCR;
 829                 oldbuf = ip->tokbuf;
 830                 ip->tokbuf = mprRealloc(ip->tokbuf, ip->tokSize);
 831                 if (ip->tokbuf == 0) {
 832                         ejsError(ep, "Token too big");
 833                         return -1;
 834                 }
 835                 ip->tokEndp += (int) ((uchar*) ip->tokbuf - oldbuf);
 836                 ip->tokServp += (int) ((uchar*) ip->tokbuf - oldbuf);
 837                 ep->token += (int) ((uchar*) ip->tokbuf - oldbuf);
 838         }
 839         *ip->tokEndp++ = c;
 840         *ip->tokEndp = '\0';
 841
 842         return 0;
 843 }
 844
 845 /******************************************************************************/
 846 /*
 847  *      Get another input character
 848  */
 849
 850 static int inputGetc(Ejs *ep)
 851 {
 852         EjsInput        *ip;
 853         int                     c;
 854
 855         mprAssert(ep);
 856         ip = ep->input;
 857
 858         if (ip->scriptSize <= 0) {
 859                 return -1;
 860         }
 861
 862         c = (uchar) (*ip->scriptServp++);
 863         ip->scriptSize--;
 864
 865         /*
 866          *      For debugging, accumulate the line number and the currenly parsed line
 867          */
 868         if (c == '\n') {
 869 #if BLD_DEBUG && 0
 870                 if (ip->lineColumn > 0) {
 871                         printf("PARSED: %s\n", ip->line);
 872                 }
 873 #endif
 874                 ip->lineNumber++;
 875                 ip->lineColumn = 0;
 876         } else {
 877                 if ((ip->lineColumn + 2) >= ip->lineLength) {
 878                         ip->lineLength += 80;
 879                         ip->line = mprRealloc(ip->line, ip->lineLength * sizeof(char));
 880                 }
 881                 ip->line[ip->lineColumn++] = c;
 882                 ip->line[ip->lineColumn] = '\0';
 883         }
 884         return c;
 885 }
 886
 887 /******************************************************************************/
 888 /*
 889  *      Putback a character onto the input queue
 890  */
 891
 892 static void inputPutback(Ejs *ep, int c)
 893 {
 894         EjsInput        *ip;
 895
 896         mprAssert(ep);
 897
 898         if (c != 0) {
 899                 ip = ep->input;
 900                 *--ip->scriptServp = c;
 901                 ip->scriptSize++;
 902                 ip->lineColumn--;
 903                 ip->line[ip->lineColumn] = '\0';
 904         }
 905 }
 906
 907 /******************************************************************************/
 908
 909 #else
 910 void ejsLexDummy() {}
 911
 912 /******************************************************************************/
 913 #endif /* BLD_FEATURE_EJS */
 914
 915 /*
 916  * Local variables:
 917  * tab-width: 4
 918  * c-basic-offset: 4
 919  * End:
 920  * vim:tw=78
 921  * vim600: sw=4 ts=4 fdm=marker
 922  * vim<600: sw=4 ts=4
 923  */