source4/lib/appweb/ejs/ejsLex.c

   1 /*
   2  *      @file   ejsLex.c
   3  *      @brief  EJS Lexical Analyser
   4  *      @overview EJS lexical analyser. This implementes a lexical analyser
   5  *              for a subset of the JavaScript language.
   6  */
   7 /********************************* Copyright **********************************/
   8 /*
   9  *      @copy   default.g
  10  *
  11  *      Copyright (c) Mbedthis Software LLC, 2003-2005. All Rights Reserved.
  12  *      Portions Copyright (c) GoAhead Software, 1995-2000. All Rights Reserved.
  13  *
  14  *      This software is distributed under commercial and open source licenses.
  15  *      You may use the GPL open source license described below or you may acquire
  16  *      a commercial license from Mbedthis Software. You agree to be fully bound
  17  *      by the terms of either license. Consult the LICENSE.TXT distributed with
  18  *      this software for full details.
  19  *
  20  *      This software is open source; you can redistribute it and/or modify it
  21  *      under the terms of the GNU General Public License as published by the
  22  *      Free Software Foundation; either version 2 of the License, or (at your
  23  *      option) any later version. See the GNU General Public License for more
  24  *      details at: http://www.mbedthis.com/downloads/gplLicense.html
  25  *
  26  *      This program is distributed WITHOUT ANY WARRANTY; without even the
  27  *      implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  28  *
  29  *      This GPL license does NOT permit incorporating this software into
  30  *      proprietary programs. If you are unable to comply with the GPL, you must
  31  *      acquire a commercial license to use this software. Commercial licenses
  32  *      for this software and support services are available from Mbedthis
  33  *      Software at http://www.mbedthis.com
  34  *
  35  *      @end
  36  */
  37 /********************************** Includes **********************************/
  38
  39 #include        "ejsInternal.h"
  40
  41 #if BLD_FEATURE_EJS
  42
  43 /****************************** Forward Declarations **************************/
  44
  45 static int              getLexicalToken(Ejs *ep, int state);
  46 static int              tokenAddChar(Ejs *ep, int c);
  47 static int              inputGetc(Ejs *ep);
  48 static void             inputPutback(Ejs *ep, int c);
  49 static int              charConvert(Ejs *ep, int base, int maxDig);
  50
  51 /************************************* Code ***********************************/
  52 /*
  53  *      Open a new input script
  54  */
  55
  56 int ejsLexOpenScript(Ejs *ep, char *script)
  57 {
  58         EjsInput        *ip;
  59
  60         mprAssert(ep);
  61         mprAssert(script);
  62
  63         if ((ep->input = mprMalloc(sizeof(EjsInput))) == NULL) {
  64                 return -1;
  65         }
  66         ip = ep->input;
  67         memset(ip, 0, sizeof(*ip));
  68
  69 /*
  70  *      Create the parse token buffer and script buffer
  71  */
  72         ip->tokbuf = mprMalloc(EJS_PARSE_INCR);
  73         ip->tokSize = EJS_PARSE_INCR;
  74         ip->tokServp = ip->tokbuf;
  75         ip->tokEndp = ip->tokbuf;
  76
  77         ip->script = mprStrdup(script);
  78         ip->scriptSize = strlen(script);
  79         ip->scriptServp = ip->script;
  80
  81         ip->lineNumber = 1;
  82         ip->lineLength = 0;
  83         ip->lineColumn = 0;
  84         ip->line = NULL;
  85
  86         ip->putBackIndex = -1;
  87
  88         return 0;
  89 }
  90
  91 /******************************************************************************/
  92 /*
  93  *      Close the input script
  94  */
  95
  96 void ejsLexCloseScript(Ejs *ep)
  97 {
  98         EjsInput        *ip;
  99         int                     i;
 100
 101         mprAssert(ep);
 102
 103         ip = ep->input;
 104         mprAssert(ip);
 105
 106         for (i = 0; i < EJS_TOKEN_STACK; i++) {
 107                 mprFree(ip->putBack[i].token);
 108                 ip->putBack[i].token = 0;
 109         }
 110
 111         mprFree(ip->line);
 112         mprFree(ip->tokbuf);
 113         mprFree(ip->script);
 114
 115         mprFree(ip);
 116 }
 117
 118 /******************************************************************************/
 119 /*
 120  *      Initialize an input state structure
 121  */
 122
 123 int ejsInitInputState(EjsInput *ip)
 124 {
 125         mprAssert(ip);
 126
 127         memset(ip, 0, sizeof(*ip));
 128         ip->putBackIndex = -1;
 129
 130         return 0;
 131 }
 132 /******************************************************************************/
 133 /*
 134  *      Save the input state
 135  */
 136
 137 void ejsLexSaveInputState(Ejs *ep, EjsInput *state)
 138 {
 139         EjsInput        *ip;
 140         int                     i;
 141
 142         mprAssert(ep);
 143
 144         ip = ep->input;
 145         mprAssert(ip);
 146
 147         *state = *ip;
 148
 149         for (i = 0; i < ip->putBackIndex; i++) {
 150                 state->putBack[i].token = mprStrdup(ip->putBack[i].token);
 151                 state->putBack[i].id = ip->putBack[i].id;
 152         }
 153         for (; i < EJS_TOKEN_STACK; i++) {
 154                 state->putBack[i].token = 0;
 155         }
 156
 157         state->line = mprMalloc(ip->lineLength);
 158         mprStrcpy(state->line, ip->lineLength, ip->line);
 159
 160         state->lineColumn = ip->lineColumn;
 161         state->lineNumber = ip->lineNumber;
 162         state->lineLength = ip->lineLength;
 163 }
 164
 165 /******************************************************************************/
 166 /*
 167  *      Restore the input state
 168  */
 169
 170 void ejsLexRestoreInputState(Ejs *ep, EjsInput *state)
 171 {
 172         EjsInput        *ip;
 173         int                     i;
 174
 175         mprAssert(ep);
 176         mprAssert(state);
 177
 178         ip = ep->input;
 179         mprAssert(ip);
 180
 181         ip->tokbuf = state->tokbuf;
 182         ip->tokServp = state->tokServp;
 183         ip->tokEndp = state->tokEndp;
 184         ip->tokSize = state->tokSize;
 185
 186         ip->script = state->script;
 187         ip->scriptServp = state->scriptServp;
 188         ip->scriptSize = state->scriptSize;
 189
 190         ip->putBackIndex = state->putBackIndex;
 191         for (i = 0; i < ip->putBackIndex; i++) {
 192                 mprFree(ip->putBack[i].token);
 193                 ip->putBack[i].id = state->putBack[i].id;
 194                 ip->putBack[i].token = mprStrdup(state->putBack[i].token);
 195         }
 196
 197         mprFree(ip->line);
 198         ip->line = mprMalloc(state->lineLength);
 199         mprStrcpy(ip->line, state->lineLength, state->line);
 200
 201         ip->lineColumn = state->lineColumn;
 202         ip->lineNumber = state->lineNumber;
 203         ip->lineLength = state->lineLength;
 204 }
 205
 206 /******************************************************************************/
 207 /*
 208  *      Free a saved input state
 209  */
 210
 211 void ejsLexFreeInputState(Ejs *ep, EjsInput *state)
 212 {
 213         int                     i;
 214
 215         mprAssert(ep);
 216         mprAssert(state);
 217
 218         for (i = 0; i < EJS_TOKEN_STACK; i++) {
 219                 mprFree(state->putBack[i].token);
 220         }
 221         state->putBackIndex = -1;
 222         mprFree(state->line);
 223         state->lineLength = 0;
 224         state->lineColumn = 0;
 225 }
 226
 227 /******************************************************************************/
 228 /*
 229  *      Get the next EJS token
 230  */
 231
 232 int ejsLexGetToken(Ejs *ep, int state)
 233 {
 234         mprAssert(ep);
 235
 236         ep->tid = getLexicalToken(ep, state);
 237         return ep->tid;
 238 }
 239
 240 /******************************************************************************/
 241
 242 /*
 243  *      Check for reserved words "if", "else", "var", "for", "foreach",
 244  *      "delete", "function", and "return". "new", "in" and "function"
 245  *      done below. "true", "false", "null", "undefined" are handled
 246  *      as global objects.
 247  *
 248  *      Other reserved words not supported:
 249  *              "break", "case", "catch", "continue", "default", "do",
 250  *              "finally", "instanceof", "switch", "this", "throw", "try",
 251  *              "typeof", "while", "with"
 252  *
 253  *      ECMA extensions reserved words (not supported):
 254  *              "abstract", "boolean", "byte", "char", "class", "const",
 255  *              "debugger", "double", "enum", "export", "extends",
 256  *              "final", "float", "goto", "implements", "import", "int",
 257  *              "interface", "long", "native", "package", "private",
 258  *              "protected", "public", "short", "static", "super",
 259  *              "synchronized", "throws", "transient", "volatile"
 260  */
 261
 262 static int checkReservedWord(Ejs *ep, int state, int c, int tid)
 263 {
 264         if (state == EJS_STATE_STMT) {
 265                 if (strcmp(ep->token, "if") == 0) {
 266                         inputPutback(ep, c);
 267                         return EJS_TOK_IF;
 268                 } else if (strcmp(ep->token, "else") == 0) {
 269                         inputPutback(ep, c);
 270                         return EJS_TOK_ELSE;
 271                 } else if (strcmp(ep->token, "var") == 0) {
 272                         inputPutback(ep, c);
 273                         return EJS_TOK_VAR;
 274                 } else if (strcmp(ep->token, "for") == 0) {
 275                         inputPutback(ep, c);
 276                         return EJS_TOK_FOR;
 277                 } else if (strcmp(ep->token, "delete") == 0) {
 278                         inputPutback(ep, c);
 279                         return EJS_TOK_DELETE;
 280                 } else if (strcmp(ep->token, "function") == 0) {
 281                         inputPutback(ep, c);
 282                         return EJS_TOK_FUNCTION;
 283                 } else if (strcmp(ep->token, "return") == 0) {
 284                         if ((c == ';') || (c == '(')) {
 285                                 inputPutback(ep, c);
 286                         }
 287                         return EJS_TOK_RETURN;
 288                 }
 289         } else if (state == EJS_STATE_EXPR) {
 290                 if (strcmp(ep->token, "new") == 0) {
 291                         inputPutback(ep, c);
 292                         return EJS_TOK_NEW;
 293                 } else if (strcmp(ep->token, "in") == 0) {
 294                         inputPutback(ep, c);
 295                         return EJS_TOK_IN;
 296                 } else if (strcmp(ep->token, "function") == 0) {
 297                         inputPutback(ep, c);
 298                         return EJS_TOK_FUNCTION;
 299                 }
 300         }
 301         return tid;
 302 }
 303
 304 /******************************************************************************/
 305 /*
 306  *      Get the next EJS token
 307  */
 308
 309 static int getLexicalToken(Ejs *ep, int state)
 310 {
 311         MprType         type;
 312         EjsInput        *ip;
 313         int                     done, tid, c, quote, style, idx;
 314
 315         mprAssert(ep);
 316         ip = ep->input;
 317         mprAssert(ip);
 318
 319         ep->tid = -1;
 320         tid = -1;
 321         type = BLD_FEATURE_NUM_TYPE_ID;
 322
 323         /*
 324          *      Use a putback tokens first. Don't free strings as caller needs access.
 325          */
 326         if (ip->putBackIndex >= 0) {
 327                 idx = ip->putBackIndex;
 328                 tid = ip->putBack[idx].id;
 329                 ep->token = (char*) ip->putBack[idx].token;
 330                 tid = checkReservedWord(ep, state, 0, tid);
 331                 ip->putBackIndex--;
 332                 return tid;
 333         }
 334         ep->token = ip->tokServp = ip->tokEndp = ip->tokbuf;
 335         *ip->tokServp = '\0';
 336
 337         if ((c = inputGetc(ep)) < 0) {
 338                 return EJS_TOK_EOF;
 339         }
 340
 341         /*
 342          *      Main lexical analyser
 343          */
 344         for (done = 0; !done; ) {
 345                 switch (c) {
 346                 case -1:
 347                         return EJS_TOK_EOF;
 348
 349                 case ' ':
 350                 case '\t':
 351                 case '\r':
 352                         do {
 353                                 if ((c = inputGetc(ep)) < 0)
 354                                         break;
 355                         } while (c == ' ' || c == '\t' || c == '\r');
 356                         break;
 357
 358                 case '\n':
 359                         return EJS_TOK_NEWLINE;
 360
 361                 case '(':
 362                         tokenAddChar(ep, c);
 363                         return EJS_TOK_LPAREN;
 364
 365                 case ')':
 366                         tokenAddChar(ep, c);
 367                         return EJS_TOK_RPAREN;
 368
 369                 case '[':
 370                         tokenAddChar(ep, c);
 371                         return EJS_TOK_LBRACKET;
 372
 373                 case ']':
 374                         tokenAddChar(ep, c);
 375                         return EJS_TOK_RBRACKET;
 376
 377                 case '.':
 378                         tokenAddChar(ep, c);
 379                         return EJS_TOK_PERIOD;
 380
 381                 case '{':
 382                         tokenAddChar(ep, c);
 383                         return EJS_TOK_LBRACE;
 384
 385                 case '}':
 386                         tokenAddChar(ep, c);
 387                         return EJS_TOK_RBRACE;
 388
 389                 case '+':
 390                         if ((c = inputGetc(ep)) < 0) {
 391                                 ejsError(ep, "Syntax Error");
 392                                 return EJS_TOK_ERR;
 393                         }
 394                         if (c != '+' ) {
 395                                 inputPutback(ep, c);
 396                                 tokenAddChar(ep, EJS_EXPR_PLUS);
 397                                 return EJS_TOK_EXPR;
 398                         }
 399                         tokenAddChar(ep, EJS_EXPR_INC);
 400                         return EJS_TOK_INC_DEC;
 401
 402                 case '-':
 403                         if ((c = inputGetc(ep)) < 0) {
 404                                 ejsError(ep, "Syntax Error");
 405                                 return EJS_TOK_ERR;
 406                         }
 407                         if (c != '-' ) {
 408                                 inputPutback(ep, c);
 409                                 tokenAddChar(ep, EJS_EXPR_MINUS);
 410                                 return EJS_TOK_EXPR;
 411                         }
 412                         tokenAddChar(ep, EJS_EXPR_DEC);
 413                         return EJS_TOK_INC_DEC;
 414
 415                 case '*':
 416                         tokenAddChar(ep, EJS_EXPR_MUL);
 417                         return EJS_TOK_EXPR;
 418
 419                 case '%':
 420                         tokenAddChar(ep, EJS_EXPR_MOD);
 421                         return EJS_TOK_EXPR;
 422
 423                 case '/':
 424                         /*
 425                          *      Handle the division operator and comments
 426                          */
 427                         if ((c = inputGetc(ep)) < 0) {
 428                                 ejsError(ep, "Syntax Error");
 429                                 return EJS_TOK_ERR;
 430                         }
 431                         if (c != '*' && c != '/') {
 432                                 inputPutback(ep, c);
 433                                 tokenAddChar(ep, EJS_EXPR_DIV);
 434                                 return EJS_TOK_EXPR;
 435                         }
 436                         style = c;
 437                         /*
 438                          *      Eat comments. Both C and C++ comment styles are supported.
 439                          */
 440                         while (1) {
 441                                 if ((c = inputGetc(ep)) < 0) {
 442                                         if (style == '/') {
 443                                                 return EJS_TOK_EOF;
 444                                         }
 445                                         ejsError(ep, "Syntax Error");
 446                                         return EJS_TOK_ERR;
 447                                 }
 448                                 if (c == '\n' && style == '/') {
 449                                         break;
 450                                 } else if (c == '*') {
 451                                         c = inputGetc(ep);
 452                                         if (style == '/') {
 453                                                 if (c == '\n') {
 454                                                         break;
 455                                                 }
 456                                         } else {
 457                                                 if (c == '/') {
 458                                                         break;
 459                                                 }
 460                                         }
 461                                 }
 462                         }
 463                         /*
 464                          *      Continue looking for a token, so get the next character
 465                          */
 466                         if ((c = inputGetc(ep)) < 0) {
 467                                 return EJS_TOK_EOF;
 468                         }
 469                         break;
 470
 471                 case '<':                                                                       /* < and <= */
 472                         if ((c = inputGetc(ep)) < 0) {
 473                                 ejsError(ep, "Syntax Error");
 474                                 return EJS_TOK_ERR;
 475                         }
 476                         if (c == '<') {
 477                                 tokenAddChar(ep, EJS_EXPR_LSHIFT);
 478                                 return EJS_TOK_EXPR;
 479                         } else if (c == '=') {
 480                                 tokenAddChar(ep, EJS_EXPR_LESSEQ);
 481                                 return EJS_TOK_EXPR;
 482                         }
 483                         tokenAddChar(ep, EJS_EXPR_LESS);
 484                         inputPutback(ep, c);
 485                         return EJS_TOK_EXPR;
 486
 487                 case '>':                                                                       /* > and >= */
 488                         if ((c = inputGetc(ep)) < 0) {
 489                                 ejsError(ep, "Syntax Error");
 490                                 return EJS_TOK_ERR;
 491                         }
 492                         if (c == '>') {
 493                                 tokenAddChar(ep, EJS_EXPR_RSHIFT);
 494                                 return EJS_TOK_EXPR;
 495                         } else if (c == '=') {
 496                                 tokenAddChar(ep, EJS_EXPR_GREATEREQ);
 497                                 return EJS_TOK_EXPR;
 498                         }
 499                         tokenAddChar(ep, EJS_EXPR_GREATER);
 500                         inputPutback(ep, c);
 501                         return EJS_TOK_EXPR;
 502
 503                 case '=':                                                                       /* "==" */
 504                         if ((c = inputGetc(ep)) < 0) {
 505                                 ejsError(ep, "Syntax Error");
 506                                 return EJS_TOK_ERR;
 507                         }
 508                         if (c == '=') {
 509                                 tokenAddChar(ep, EJS_EXPR_EQ);
 510                                 return EJS_TOK_EXPR;
 511                         }
 512                         inputPutback(ep, c);
 513                         return EJS_TOK_ASSIGNMENT;
 514
 515                 case '!':                                                                       /* "!=" or "!"*/
 516                         if ((c = inputGetc(ep)) < 0) {
 517                                 ejsError(ep, "Syntax Error");
 518                                 return EJS_TOK_ERR;
 519                         }
 520                         if (c == '=') {
 521                                 tokenAddChar(ep, EJS_EXPR_NOTEQ);
 522                                 return EJS_TOK_EXPR;
 523                         }
 524                         inputPutback(ep, c);
 525                         tokenAddChar(ep, EJS_EXPR_BOOL_COMP);
 526                         return EJS_TOK_EXPR;
 527
 528                 case ';':
 529                         tokenAddChar(ep, c);
 530                         return EJS_TOK_SEMI;
 531
 532                 case ',':
 533                         tokenAddChar(ep, c);
 534                         return EJS_TOK_COMMA;
 535
 536                 case '|':                                                                       /* "||" */
 537                         if ((c = inputGetc(ep)) < 0 || c != '|') {
 538                                 ejsError(ep, "Syntax Error");
 539                                 return EJS_TOK_ERR;
 540                         }
 541                         tokenAddChar(ep, EJS_COND_OR);
 542                         return EJS_TOK_LOGICAL;
 543
 544                 case '&':                                                                       /* "&&" */
 545                         if ((c = inputGetc(ep)) < 0 || c != '&') {
 546                                 ejsError(ep, "Syntax Error");
 547                                 return EJS_TOK_ERR;
 548                         }
 549                         tokenAddChar(ep, EJS_COND_AND);
 550                         return EJS_TOK_LOGICAL;
 551
 552                 case '\"':                                                                      /* String quote */
 553                 case '\'':
 554                         quote = c;
 555                         if ((c = inputGetc(ep)) < 0) {
 556                                 ejsError(ep, "Syntax Error");
 557                                 return EJS_TOK_ERR;
 558                         }
 559
 560                         while (c != quote) {
 561                                 /*
 562                                  *      Check for escape sequence characters
 563                                  */
 564                                 if (c == '\\') {
 565                                         c = inputGetc(ep);
 566
 567                                         if (isdigit(c)) {
 568                                                 /*
 569                                                  *      Octal support, \101 maps to 65 = 'A'. Put first
 570                                                  *      char back so converter will work properly.
 571                                                  */
 572                                                 inputPutback(ep, c);
 573                                                 c = charConvert(ep, 8, 3);
 574
 575                                         } else {
 576                                                 switch (c) {
 577                                                 case 'n':
 578                                                         c = '\n'; break;
 579                                                 case 'b':
 580                                                         c = '\b'; break;
 581                                                 case 'f':
 582                                                         c = '\f'; break;
 583                                                 case 'r':
 584                                                         c = '\r'; break;
 585                                                 case 't':
 586                                                         c = '\t'; break;
 587                                                 case 'x':
 588                                                         /*
 589                                                          *      Hex support, \x41 maps to 65 = 'A'
 590                                                          */
 591                                                         c = charConvert(ep, 16, 2);
 592                                                         break;
 593                                                 case 'u':
 594                                                         /*
 595                                                          *      Unicode support, \x0401 maps to 65 = 'A'
 596                                                          */
 597                                                         c = charConvert(ep, 16, 2);
 598                                                         c = c*16 + charConvert(ep, 16, 2);
 599
 600                                                         break;
 601                                                 case '\'':
 602                                                 case '\"':
 603                                                 case '\\':
 604                                                         break;
 605                                                 default:
 606                                                         ejsError(ep, "Invalid Escape Sequence");
 607                                                         return EJS_TOK_ERR;
 608                                                 }
 609                                         }
 610                                         if (tokenAddChar(ep, c) < 0) {
 611                                                 return EJS_TOK_ERR;
 612                                         }
 613                                 } else {
 614                                         if (tokenAddChar(ep, c) < 0) {
 615                                                 return EJS_TOK_ERR;
 616                                         }
 617                                 }
 618                                 if ((c = inputGetc(ep)) < 0) {
 619                                         ejsError(ep, "Unmatched Quote");
 620                                         return EJS_TOK_ERR;
 621                                 }
 622                         }
 623                         return EJS_TOK_LITERAL;
 624
 625                 case '0':
 626                         if (tokenAddChar(ep, c) < 0) {
 627                                 return EJS_TOK_ERR;
 628                         }
 629                         if ((c = inputGetc(ep)) < 0) {
 630                                 break;
 631                         }
 632                         if (tolower(c) == 'x') {
 633                                 if (tokenAddChar(ep, c) < 0) {
 634                                         return EJS_TOK_ERR;
 635                                 }
 636                                 if ((c = inputGetc(ep)) < 0) {
 637                                         break;
 638                                 }
 639                         }
 640                         if (! isdigit(c)) {
 641 #if BLD_FEATURE_FLOATING_POINT
 642                                 if (c == '.' || tolower(c) == 'e' || c == '+' || c == '-') {
 643                                         /* Fall through */
 644                                         type = MPR_TYPE_FLOAT;
 645                                 } else
 646 #endif
 647                                 {
 648                                         mprDestroyVar(&ep->tokenNumber);
 649                                         ep->tokenNumber = mprParseVar(ep->token, type);
 650                                         inputPutback(ep, c);
 651                                         return EJS_TOK_NUMBER;
 652                                 }
 653                         }
 654                         /* Fall through to get more digits */
 655
 656                 case '1': case '2': case '3': case '4':
 657                 case '5': case '6': case '7': case '8': case '9':
 658                         do {
 659                                 if (tokenAddChar(ep, c) < 0) {
 660                                         return EJS_TOK_ERR;
 661                                 }
 662                                 if ((c = inputGetc(ep)) < 0) {
 663                                         break;
 664                                 }
 665 #if BLD_FEATURE_FLOATING_POINT
 666                                 if (c == '.' || tolower(c) == 'e' || tolower(c) == 'f') {
 667                                         type = MPR_TYPE_FLOAT;
 668                                 }
 669                         } while (isdigit(c) || c == '.' || tolower(c) == 'e' || tolower(c) == 'f' ||
 670                                 ((type == MPR_TYPE_FLOAT) && (c == '+' || c == '-')));
 671 #else
 672                         } while (isdigit(c));
 673 #endif
 674
 675                         mprDestroyVar(&ep->tokenNumber);
 676                         ep->tokenNumber = mprParseVar(ep->token, type);
 677                         inputPutback(ep, c);
 678                         return EJS_TOK_NUMBER;
 679
 680                 default:
 681                         /*
 682                          *      Identifiers or a function names
 683                          */
 684                         while (1) {
 685                                 if (c == '\\') {
 686                                         if ((c = inputGetc(ep)) < 0) {
 687                                                 break;
 688                                         }
 689                                         if (c == '\n' || c == '\r') {
 690                                                 break;
 691                                         }
 692                                 } else if (tokenAddChar(ep, c) < 0) {
 693                                                 break;
 694                                 }
 695                                 if ((c = inputGetc(ep)) < 0) {
 696                                         break;
 697                                 }
 698                                 if (!isalnum(c) && c != '$' && c != '_' && c != '\\') {
 699                                         break;
 700                                 }
 701                         }
 702                         if (*ep->token == '\0') {
 703                                 c = inputGetc(ep);
 704                                 break;
 705                         }
 706                         if (! isalpha((int) *ep->token) && *ep->token != '$' &&
 707                                         *ep->token != '_') {
 708                                 ejsError(ep, "Invalid identifier %s", ep->token);
 709                                 return EJS_TOK_ERR;
 710                         }
 711
 712                         tid = checkReservedWord(ep, state, c, EJS_TOK_ID);
 713                         if (tid != EJS_TOK_ID) {
 714                                 return tid;
 715                         }
 716
 717                         /*
 718                          *      Skip white space after token to find out whether this is
 719                          *      a function or not.
 720                          */
 721                         while (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
 722                                 if ((c = inputGetc(ep)) < 0)
 723                                         break;
 724                         }
 725
 726                         tid = EJS_TOK_ID;
 727                         done++;
 728                 }
 729         }
 730
 731         /*
 732          *      Putback the last extra character for next time
 733          */
 734         inputPutback(ep, c);
 735         return tid;
 736 }
 737
 738 /******************************************************************************/
 739 /*
 740  *      Convert a hex or octal character back to binary, return original char if
 741  *      not a hex digit
 742  */
 743
 744 static int charConvert(Ejs *ep, int base, int maxDig)
 745 {
 746         int             i, c, lval, convChar;
 747
 748         lval = 0;
 749         for (i = 0; i < maxDig; i++) {
 750                 if ((c = inputGetc(ep)) < 0) {
 751                         break;
 752                 }
 753                 /*
 754                  *      Initialize to out of range value
 755                  */
 756                 convChar = base;
 757                 if (isdigit(c)) {
 758                         convChar = c - '0';
 759                 } else if (c >= 'a' && c <= 'f') {
 760                         convChar = c - 'a' + 10;
 761                 } else if (c >= 'A' && c <= 'F') {
 762                         convChar = c - 'A' + 10;
 763                 }
 764                 /*
 765                  *      If unexpected character then return it to buffer.
 766                  */
 767                 if (convChar >= base) {
 768                         inputPutback(ep, c);
 769                         break;
 770                 }
 771                 lval = (lval * base) + convChar;
 772         }
 773         return lval;
 774 }
 775
 776 /******************************************************************************/
 777 /*
 778  *      Putback the last token read. Accept at most one push back token.
 779  */
 780
 781 void ejsLexPutbackToken(Ejs *ep, int tid, char *string)
 782 {
 783         EjsInput        *ip;
 784         int                     idx;
 785
 786         mprAssert(ep);
 787         ip = ep->input;
 788         mprAssert(ip);
 789
 790         ip->putBackIndex += 1;
 791         idx = ip->putBackIndex;
 792         ip->putBack[idx].id = tid;
 793
 794         if (ip->putBack[idx].token) {
 795                 if (ip->putBack[idx].token == string) {
 796                         return;
 797                 }
 798                 mprFree(ip->putBack[idx].token);
 799         }
 800         ip->putBack[idx].token = mprStrdup(string);
 801 }
 802
 803 /******************************************************************************/
 804 /*
 805  *      Add a character to the token buffer
 806  */
 807
 808 static int tokenAddChar(Ejs *ep, int c)
 809 {
 810         EjsInput        *ip;
 811         uchar           *oldbuf;
 812
 813         mprAssert(ep);
 814         ip = ep->input;
 815         mprAssert(ip);
 816
 817         if (ip->tokEndp >= &ip->tokbuf[ip->tokSize - 1]) {
 818                 ip->tokSize += EJS_PARSE_INCR;
 819                 oldbuf = ip->tokbuf;
 820                 ip->tokbuf = mprRealloc(ip->tokbuf, ip->tokSize);
 821                 if (ip->tokbuf == 0) {
 822                         ejsError(ep, "Token too big");
 823                         return -1;
 824                 }
 825                 ip->tokEndp += (int) ((uchar*) ip->tokbuf - oldbuf);
 826                 ip->tokServp += (int) ((uchar*) ip->tokbuf - oldbuf);
 827                 ep->token += (int) ((uchar*) ip->tokbuf - oldbuf);
 828         }
 829         *ip->tokEndp++ = c;
 830         *ip->tokEndp = '\0';
 831
 832         return 0;
 833 }
 834
 835 /******************************************************************************/
 836 /*
 837  *      Get another input character
 838  */
 839
 840 static int inputGetc(Ejs *ep)
 841 {
 842         EjsInput        *ip;
 843         int                     c;
 844
 845         mprAssert(ep);
 846         ip = ep->input;
 847
 848         if (ip->scriptSize <= 0) {
 849                 return -1;
 850         }
 851
 852         c = (uchar) (*ip->scriptServp++);
 853         ip->scriptSize--;
 854
 855         /*
 856          *      For debugging, accumulate the line number and the currenly parsed line
 857          */
 858         if (c == '\n') {
 859 #if BLD_DEBUG && 0
 860                 if (ip->lineColumn > 0) {
 861                         printf("PARSED: %s\n", ip->line);
 862                 }
 863 #endif
 864                 ip->lineNumber++;
 865                 ip->lineColumn = 0;
 866         } else {
 867                 if ((ip->lineColumn + 2) >= ip->lineLength) {
 868                         ip->lineLength += 80;
 869                         ip->line = mprRealloc(ip->line, ip->lineLength * sizeof(char));
 870                 }
 871                 ip->line[ip->lineColumn++] = c;
 872                 ip->line[ip->lineColumn] = '\0';
 873         }
 874         return c;
 875 }
 876
 877 /******************************************************************************/
 878 /*
 879  *      Putback a character onto the input queue
 880  */
 881
 882 static void inputPutback(Ejs *ep, int c)
 883 {
 884         EjsInput        *ip;
 885
 886         mprAssert(ep);
 887
 888         if (c != 0) {
 889                 ip = ep->input;
 890                 *--ip->scriptServp = c;
 891                 ip->scriptSize++;
 892                 ip->lineColumn--;
 893                 ip->line[ip->lineColumn] = '\0';
 894         }
 895 }
 896
 897 /******************************************************************************/
 898
 899 #else
 900 void ejsLexDummy() {}
 901
 902 /******************************************************************************/
 903 #endif /* BLD_FEATURE_EJS */
 904
 905 /*
 906  * Local variables:
 907  * tab-width: 4
 908  * c-basic-offset: 4
 909  * End:
 910  * vim:tw=78
 911  * vim600: sw=4 ts=4 fdm=marker
 912  * vim<600: sw=4 ts=4
 913  */