text: /*
  ** $Id: llex.c,v 2.20 2006/03/09 18:14:31 roberto Exp $
  ** Lexical Analyzer
  ** See Copyright Notice in lua.h
  */
  
  
  #include <ctype.h>
  #include <locale.h>
  #include <string.h>
  
  #define llex_c
  #define LUA_CORE
  
  #include "lua.h"
  
  #include "ldo.h"
  #include "llex.h"
  #include "lobject.h"
  #include "lparser.h"
  #include "lstate.h"
  #include "lstring.h"
  #include "ltable.h"
  #include "lzio.h"
  #include <assert.h>
  
  NAMESPACE_LUA_BEGIN
  
  static int next(LexState *ls)
  {
    union
    {
  	lua_WChar w;
  	unsigned char b[2];
    } s;
    int c = zgetc(ls->z);
    if (c == EOZ) {
      ls->current = EOZ;
  	return EOZ;
    }
    if (!ls->z->isWide) {
      ls->current = c;
      return ls->current;
    }
  
    s.b[0] = (unsigned char)c;
    c = zgetc(ls->z);
    if (c == EOZ) {
      ls->current = EOZ;
      return EOZ;
    }
    s.b[1] = (unsigned char)c;
    ls->current = s.w;
    return ls->current;
  }
  
  
  
  
  
  
  #define currIsNewline(ls)	(ls->current == '\n' || ls->current == '\r')
  
  
  /* ORDER RESERVED */
  const char *const luaX_tokens [] = {
      "and", "break", "continue", "do", "else", "elseif",
      "end", "false", "for", "function", "if",
      "in", "local", "nil", "not", "or", "repeat",
      "return", "then", "true", "until", "while",
      "..", "...", "==", ">=", "<=", "~=",
      "<number>", "<name>", "<string>", "<wstring>", "<<", ">>", "^^", "<eof>",
      NULL
  };
  
  
  #define save_and_next(ls) (save(ls, ls->current), next(ls))
  
  
  static void save (LexState *ls, int c) {
    Mbuffer *b = ls->buff;
    if (b->n + 1 > b->buffsize) {
      size_t newsize;
      if (b->buffsize >= MAX_SIZET/2)
        luaX_lexerror(ls, "lexical element too long", 0);
      newsize = b->buffsize * 2;
      luaZ_resizebuffer(ls->L, b, newsize);
    }
    b->buffer[b->n++] = cast(char, c);
  }
  
  
  #define wsave_and_next(ls) (wsave(ls, ls->current), next(ls))
  
  
  static void wsave (LexState *ls, int c) {
    Mbuffer *b = ls->buff;
    if (b->n + 2 > b->buffsize) {
      size_t newsize;
      if (b->buffsize >= MAX_SIZET/2)
        luaX_lexerror(ls, "lexical element too long", 0);
      newsize = b->buffsize * 4;
      luaZ_resizebuffer(ls->L, b, newsize);
    }
    *(lua_WChar*)(&b->buffer[b->n]) = cast(lua_WChar, c);
    b->n += 2;
  }
  
  
  void luaX_init (lua_State *L) {
    int i;
    for (i=0; i<NUM_RESERVED; i++) {
      TString *ts = luaS_new(L, luaX_tokens[i]);
      luaS_fix(ts);  /* reserved words are never collected */
      lua_assert(strlen(luaX_tokens[i])+1 <= TOKEN_LEN);
      ts->tsv.reserved = cast_byte(i+1);  /* reserved word */
    }
  }
  
  
  #define MAXSRC          80
  
  
  const char *luaX_token2str (LexState *ls, int token) {
    if (token < FIRST_RESERVED) {
      lua_assert(token == cast(unsigned char, token));
      return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
                                luaO_pushfstring(ls->L, "%c", token);
    }
    else
      return luaX_tokens[token-FIRST_RESERVED];
  }
  
  
  static const char *txtToken (LexState *ls, int token) {
    switch (token) {
      case TK_NAME:
      case TK_STRING:
      case TK_NUMBER:
        save(ls, '\0');
        return luaZ_buffer(ls->buff);
      default:
        return luaX_token2str(ls, token);
    }
  }
  
  
  void luaX_lexerror (LexState *ls, const char *msg, int token) {
    char buff[MAXSRC];
    luaO_chunkid(buff, getstr(ls->source), MAXSRC);
    msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
    if (token)
      luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
    luaD_throw(ls->L, LUA_ERRSYNTAX);
  }
  
  
  void luaX_syntaxerror (LexState *ls, const char *msg) {
    luaX_lexerror(ls, msg, ls->t.token);
  }
  
  
  TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
    lua_State *L = ls->L;
    TString *ts = luaS_newlstr(L, str, l);
    TValue *o = luaH_setstr(L, ls->fs->h, ts);  /* entry for `str' */
    if (ttisnil(o))
      setbvalue(o, 1);  /* make sure `str' will not be collected */
    return ts;
  }
  
  
  TString *luaX_newwstring (LexState *ls, const lua_WChar *str, size_t l) {
    lua_State *L = ls->L;
    TString *ts = luaS_newlwstr(L, str, l);
    TValue *o = luaH_setwstr(L, ls->fs->h, ts);  /* entry for `str' */
    if (ttisnil(o))
      setbvalue(o, 1);  /* make sure `str' will not be collected */
    return ts;
  }
  
  
  static void inclinenumber (LexState *ls) {
    int old = ls->current;
    lua_assert(currIsNewline(ls));
    next(ls);  /* skip `\n' or `\r' */
    if (currIsNewline(ls) && ls->current != old)
      next(ls);  /* skip `\n\r' or `\r\n' */
    if (++ls->linenumber >= MAX_INT)
      luaX_syntaxerror(ls, "chunk has too many lines");
  }
  
  
  void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
    ls->decpoint = '.';
    ls->L = L;
    ls->lookahead.token = TK_EOS;  /* no look-ahead token */
    ls->z = z;
    ls->fs = NULL;
    ls->linenumber = 1;
    ls->lastline = 1;
    ls->source = source;
    luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
    next(ls);  /* read first char */
  }
  
  
  
  /*
  ** =======================================================
  ** LEXICAL ANALYZER
  ** =======================================================
  */
  
  
  
  static int check_next (LexState *ls, const char *set) {
    if (!strchr(set, ls->current))
      return 0;
    save_and_next(ls);
    return 1;
  }
  
  
  static void buffreplace (LexState *ls, char from, char to) {
    size_t n = luaZ_bufflen(ls->buff);
    char *p = luaZ_buffer(ls->buff);
    while (n--)
      if (p[n] == from) p[n] = to;
  }
  
  
  static void trydecpoint (LexState *ls, SemInfo *seminfo) {
    /* format error: try to update decimal point separator */
    struct lconv *cv = localeconv();
    char old = ls->decpoint;
    ls->decpoint = (cv ? cv->decimal_point[0] : '.');
    buffreplace(ls, old, ls->decpoint);  /* try updated decimal separator */
    if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
      /* format error with correct decimal point: no more options */
      buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
      luaX_lexerror(ls, "malformed number", TK_NUMBER);
    }
  }
  
  
  /* LUA_NUMBER */
  static void read_numeral (LexState *ls, SemInfo *seminfo) {
    lua_assert(isdigit(ls->current));
    if (ls->current == '0') {
      save_and_next(ls);
      if (ls->current == 'x') {
  	  /* Process a hex number */
  	  int ch = 0;
        int c = 0;
        int i = 0;
        int numDigits = 8;
  	  next(ls);
        do {
          ch = tolower(ls->current);
          if (isdigit(ch))
            c = 16*c + (ch-'0');
          else if (ch >= 'a' && ch <= 'f')
            c = 16*c + (ch-'a') + 10;
          next(ls);
          ch = tolower(ls->current);
        } while (++i<numDigits && (isdigit(ch) || (ch >= 'a' && ch <= 'f')));
  	  seminfo->r = c;
  	  return;
  	}
    }
  
    while (isdigit(ls->current) || ls->current == '.') {
      save_and_next(ls);
    }
    if (check_next(ls, "Ee"))  /* `E'? */
      check_next(ls, "+-");  /* optional exponent sign */
    while (isalnum(ls->current) || ls->current == '_')
      save_and_next(ls);
    save(ls, '\0');
    buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
    if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r))  /* format error? */
      trydecpoint(ls, seminfo); /* try to update decimal point separator */
  }
  
  
  static int skip_sep (LexState *ls) {
    int count = 0;
    int s = ls->current;
    lua_assert(s == '[' || s == ']');
    save_and_next(ls);
    while (ls->current == '=') {
      save_and_next(ls);
      count++;
    }
    return (ls->current == s) ? count : (-count) - 1;
  }
  
  
  static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
    int cont = 0;
    (void)(cont);  /* avoid warnings when `cont' is not used */
    save_and_next(ls);  /* skip 2nd `[' */
    if (currIsNewline(ls))  /* string starts with a newline? */
      inclinenumber(ls);  /* skip it */
    for (;;) {
      switch (ls->current) {
        case EOZ:
          luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
                                     "unfinished long comment", TK_EOS);
          break;  /* to avoid warnings */
  #if defined(LUA_COMPAT_LSTR)
        case '[': {
          if (skip_sep(ls) == sep) {
            save_and_next(ls);  /* skip 2nd `[' */
            cont++;
  #if LUA_COMPAT_LSTR == 1
            if (sep == 0)
              luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
  #endif
          }
          break;
        }
  #endif
        case ']': {
          if (skip_sep(ls) == sep) {
            save_and_next(ls);  /* skip 2nd `]' */
  #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
            cont--;
            if (sep == 0 && cont >= 0) break;
  #endif
            goto endloop;
          }
          break;
        }
        case '\n':
        case '\r': {
          save(ls, '\n');
          inclinenumber(ls);
          if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
          break;
        }
        default: {
          if (seminfo) save_and_next(ls);
          else next(ls);
        }
      }
    } endloop:
    if (seminfo)
      seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
                                       luaZ_bufflen(ls->buff) - 2*(2 + sep));
  }
  
  
  static void read_string (LexState *ls, int del, SemInfo *seminfo) {
    save_and_next(ls);
    while (ls->current != del) {
      switch (ls->current) {
        case EOZ:
          luaX_lexerror(ls, "unfinished string", TK_EOS);
          continue;  /* to avoid warnings */
        case '\n':
        case '\r':
          luaX_lexerror(ls, "unfinished string", TK_STRING);
          continue;  /* to avoid warnings */
        case '\\': {
          int c;
          next(ls);  /* do not save the `\' */
          switch (ls->current) {
            case 'a': c = '\a'; break;
            case 'b': c = '\b'; break;
            case 'f': c = '\f'; break;
            case 'n': c = '\n'; break;
            case 'r': c = '\r'; break;
            case 't': c = '\t'; break;
            case 'v': c = '\v'; break;
            case '\n':  /* go through */
            case '\r': save(ls, '\n'); inclinenumber(ls); continue;
            case EOZ: continue;  /* will raise an error next loop */
            case 'x': {
              int ch;
  			int i = 0;
  			int numDigits = 2;
              c = 0;
  			next(ls);
  			do {
                ch = tolower(ls->current);
                if (isdigit(ch))
                  c = 16*c + (ch-'0');
                else if (ch >= 'a' && ch <= 'f')
                  c = 16*c + (ch-'a') + 10;
                next(ls);
                ch = tolower(ls->current);
  			} while (++i<numDigits && (isdigit(ch) || (ch >= 'a' && ch <= 'f')));
  			save(ls, c);
              continue;
            }
            default: {
              if (!isdigit(ls->current))
                save_and_next(ls);  /* handles \\, \", \', and \? */
              else {  /* \xxx */
                int i = 0;
                c = 0;
                do {
                  c = 10*c + (ls->current-'0');
                  next(ls);
                } while (++i<3 && isdigit(ls->current));
                if (c > UCHAR_MAX)
                  luaX_lexerror(ls, "escape sequence too large", TK_STRING);
                save(ls, c);
              }
              continue;
            }
          }
          save(ls, c);
          next(ls);
          continue;
        }
        default:
          save_and_next(ls);
      }
    }
    save_and_next(ls);  /* skip delimiter */
    seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
                                     luaZ_bufflen(ls->buff) - 2);
  }
  
  
  static void read_wstring (LexState *ls, int del, SemInfo *seminfo) {
    wsave_and_next(ls);
    while (ls->current != del) {
      switch (ls->current) {
        case EOZ:
          luaX_lexerror(ls, "unfinished string", TK_EOS);
          continue;  /* to avoid warnings */
        case '\n':
        case '\r':
          luaX_lexerror(ls, "unfinished string", TK_STRING);
          continue;  /* to avoid warnings */
        case '\\': {
          int c;
          next(ls);  /* do not save the `\' */
          switch (ls->current) {
            case 'a': c = '\a'; break;
            case 'b': c = '\b'; break;
            case 'f': c = '\f'; break;
            case 'n': c = '\n'; break;
            case 'r': c = '\r'; break;
            case 't': c = '\t'; break;
            case 'v': c = '\v'; break;
            case '\n':  /* go through */
            case '\r': wsave(ls, '\n'); inclinenumber(ls); continue;
            case EOZ: continue;  /* will raise an error next loop */
            case 'x': {
              int ch;
  			int i = 0;
  			int numDigits = 4;
              c = 0;
  			next(ls);
  			do {
                ch = tolower(ls->current);
                if (isdigit(ch))
                  c = 16*c + (ch-'0');
                else if (ch >= 'a' && ch <= 'f')
                  c = 16*c + (ch-'a') + 10;
                next(ls);
                ch = tolower(ls->current);
  			} while (++i<numDigits && (isdigit(ch) || (ch >= 'a' && ch <= 'f')));
  			wsave(ls, c);
              continue;
            }
            default: {
              if (!isdigit(ls->current))
                wsave_and_next(ls);  /* handles \\, \", \', and \? */
              else {  /* \xxx */
                int i = 0;
                c = 0;
                do {
                  c = 10*c + (ls->current-'0');
                  next(ls);
                } while (++i<3 && isdigit(ls->current));
                if (c > UCHAR_MAX)
                  luaX_lexerror(ls, "escape sequence too large", TK_STRING);
                wsave(ls, c);
              }
              continue;
            }
          }
          wsave(ls, c);
          next(ls);
          continue;
        }
        default:
          wsave_and_next(ls);
      }
    }
    wsave_and_next(ls);  /* skip delimiter */
    seminfo->ts = luaX_newwstring(ls, (const lua_WChar*)(luaZ_buffer(ls->buff) + 1 * 2),
                                     (luaZ_bufflen(ls->buff) - 2 * 2) / 2 );
  }
  
  
  static int llex (LexState *ls, SemInfo *seminfo) {
    luaZ_resetbuffer(ls->buff);
    for (;;) {
      switch (ls->current) {
        case '\n':
        case '\r': {
          inclinenumber(ls);
          continue;
        }
        case '-': {
          next(ls);
          if (ls->current != '-') return '-';
          /* else is a comment */
          next(ls);
          if (ls->current == '[') {
            int sep = skip_sep(ls);
            luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
            if (sep >= 0) {
              read_long_string(ls, NULL, sep);  /* long comment */
              luaZ_resetbuffer(ls->buff);
              continue;
            }
          }
          /* else short comment */
          while (!currIsNewline(ls) && ls->current != EOZ)
            next(ls);
          continue;
        }
        case '[': {
          int sep = skip_sep(ls);
          if (sep >= 0) {
            read_long_string(ls, seminfo, sep);
            return TK_STRING;
          }
          else if (sep == -1) return '[';
          else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
        }
        case '=': {
          next(ls);
          if (ls->current != '=') return '=';
          else { next(ls); return TK_EQ; }
        }
        case '<': {
          next(ls);
          if (ls->current == '<') { next(ls); return TK_SHL; }
          if (ls->current != '=') return '<';
          else { next(ls); return TK_LE; }
        }
        case '>': {
          next(ls);
          if (ls->current == '>') { next(ls); return TK_SHR; }
          if (ls->current != '=') return '>';
          else { next(ls); return TK_GE; }
        }
        case '~': {
          next(ls);
          if (ls->current != '=') return '~';
          else { next(ls); return TK_NE; }
        }
        case '^': {
          next(ls);
          if (ls->current != '^') return '^';
          else { next(ls); return TK_XOR; }
        }
        case '"':
        case '\'': {
          read_string(ls, ls->current, seminfo);
          return TK_STRING;
        }
        case '.': {
          save_and_next(ls);
          if (check_next(ls, ".")) {
            if (check_next(ls, "."))
              return TK_DOTS;   /* ... */
            else return TK_CONCAT;   /* .. */
          }
          else if (!isdigit(ls->current)) return '.';
          else {
            read_numeral(ls, seminfo);
            return TK_NUMBER;
          }
        }
        case EOZ: {
          return TK_EOS;
        }
        default: {
          if (ls->current > 255) {
            luaX_lexerror(ls, "invalid wide char", 0);
            next(ls);
            return ls->current;  /* single-char tokens (+ - / ...) */
          }
          if (isspace(ls->current)) {
            lua_assert(!currIsNewline(ls));
            next(ls);
            continue;
          }
          else if (isdigit(ls->current)) {
            read_numeral(ls, seminfo);
            return TK_NUMBER;
          }
          else if (isalpha(ls->current) || ls->current == '_') {
            /* identifier or reserved word */
            TString *ts;
            if (ls->current == 'L') {
              next(ls);
              if (ls->current == '"' || ls->current == '\'') {
                read_wstring(ls, ls->current, seminfo);
                return TK_WSTRING;
              }
              save(ls, 'L');
            }
            /* identifier or reserved word */
            do {
              save_and_next(ls);
            } while (isalnum(ls->current) || ls->current == '_');
            ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
                                    luaZ_bufflen(ls->buff));
            if (ts->tsv.reserved > 0)  /* reserved word? */
              return ts->tsv.reserved - 1 + FIRST_RESERVED;
            else {
              seminfo->ts = ts;
              return TK_NAME;
            }
          }
          else {
            int c = ls->current;
            next(ls);
            return c;  /* single-char tokens (+ - / ...) */
          }
        }
      }
    }
  }
  
  
  void luaX_next (LexState *ls) {
    ls->lastline = ls->linenumber;
    if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
      ls->t = ls->lookahead;  /* use this one */
      ls->lookahead.token = TK_EOS;  /* and discharge it */
    }
    else
      ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
  }
  
  
  void luaX_lookahead (LexState *ls) {
    lua_assert(ls->lookahead.token == TK_EOS);
    ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  }
  
  NAMESPACE_LUA_END
  