text: // Copyright (c)1997-1999 Microsoft Corporation, All Rights Reserved
  /* copied from ..\htmed\lexer.cpp */
  
  /*++
  
    Copyright (c) 1995 Microsoft Corporation
  
    File: lexer.cpp
  
    Abstract:
          Nitty Gritty Lexer stuff
  
    Contents:
        SetValueSeen()
        IsSingleOp()
        IsWhiteSpace()
        MapToken()
        FindEndTag()
        MakeSublang()
        SetLanguage()
        FindTable()
        FindTable()
        RemoveTable()
        MakeTableSet()
        GetToken()
        IfHackComment()
        FindServerScript()
        FindEndComment()
        FindEndEntity()
        FindEntityRef()
        FindValue()
        FindEndString()
        FindTagOpen()
        FindText()
        FindNextToken()
        GetTextHint()
        GetHint()
        GetTokenLength()
        GetValueTokenLength()
        IsElementName()
        IsAttributeName()
        IsIdentifier()
        IsUnknownID()
        IsNumber()
        CColorHtml::SetTable()
        CColorHtml::InitSublanguages()
  
    History:
        2/14/97   cgomes:   Created
  
  
  --*/
  
  #include "stdafx.h"
  
  #include "resource.h"
  #include "guids.h"
  #include "token.h"
  #include "table.h"
  #include "lexer.h"
  
  UINT FindClientScriptEnd(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token);
  
  #undef ASSERT
  #define ASSERT(b) _ASSERTE(b)
  // HACK: we keep a copy of a ptr to the ASP table and sublang
  // so we can do special behavior for ASP files
  CTableSet* g_ptabASP = 0;
  PSUBLANG g_psublangASP = 0;
  
  PTABLESET g_arpTables[CV_MAX+1];
  
  // NOTE: added to handle value tokens properly.
  UINT GetValueTokenLength(LPCTSTR pchLine, UINT cbLen, UINT cbCur);
  
  // mark state transition from value -> next attribute
  inline int SetValueSeen(DWORD *plxs)
  {
      if (*plxs & inValue)
      {
          *plxs &= ~inValue;
          *plxs |= inAttribute;
          return TRUE;
      }
      else
          return FALSE;
  }
  
  // REVIEW (walts) - need better way
  inline void SetScriptLanguage(LPCTSTR pchLine, DWORD *plxs)
  {
      LPCTSTR strJavaScript       = _T("javascript");
      LPCTSTR strVBScript         = _T("vbscript");
      // triedit's special language. Its set when we convert server-side scripts into
      // client-side scripts. Its a dummy language. if we find that as language, we
      // set in ServerASP. It is reset(removed) in FindNextToken().
      LPCTSTR strServerAsp        = _T("serverasp");
  
      // language attribute may have quotes around it.
      // if it does then advance past the first quote.
      //      ex. <SCRIPT LANGUAGE="VBScript">
      if(*pchLine == L'\"')
          pchLine++;
  
      if (_tcsnicmp(pchLine, strJavaScript, lstrlen(strJavaScript)) == 0)
      {
          *plxs &= ~inVBScript;
          *plxs &= ~inServerASP;
          *plxs |= inJavaScript;
      }
      else if (_tcsnicmp(pchLine, strVBScript, lstrlen(strVBScript)) == 0)
      {
          *plxs &= ~inJavaScript;
          *plxs &= ~inServerASP;
          *plxs |= inVBScript;
      }
      else if (_tcsnicmp(pchLine, strServerAsp, lstrlen(strServerAsp)) == 0)
      {
          *plxs &= ~inJavaScript;
          *plxs &= ~inVBScript;
          *plxs |= inServerASP;
      }
  }
  
  inline BOOL IsSingleOp(HINT hint)
  {
      return ((hint >= tokOP_SINGLE) && (hint < tokOP_MAX));
  };
  
  inline BOOL IsWhiteSpace(TCHAR c)
  {
      return _istspace(c);
  };
  
  
  // NOTE: Added to handle value tokens properly
  inline IsValueChar(TCHAR ch)
  {
      // REVIEW(cgomes): specify all the invalid value characters
      return ch != _T('<') && ch != _T('>');
  };
  
  ////////////////////////////////////////////////////////////////////////////
  //
  // map parsed token to returned token
  
  // left column must be in ascending order
  static TOKEN _rgTokenMap[] =
  {
      tokName,     tokSpace,
      tokNum,      tokSpace,
      tokParEnt,   tokSpace,
      tokResName,  tokSpace,
      0,           0
  };
  
  static TOKEN MapToken(TOKEN tokClass, DWORD lxs)
  {
      if (IsSingleOp((HINT)tokClass))
          return tokOp;
      else if ((tokClass == tokTag) && (lxs & inHTXTag))
          return tokSSS;
      for (int i = 0; (_rgTokenMap[i] != 0) && (_rgTokenMap[i] >= tokClass); i += 2)
      {
          if (_rgTokenMap[i] == tokClass)
              return _rgTokenMap[i + 1];
      }
      return tokClass;
  }
  
  ////////////////////////////////////////////////////////////////////////////
  
  UINT FindEndTag(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD *plxs, TXTB & token)
  {
      ASSERT(pchLine);
      TCHAR szEnd[16];
      ELLEX * pellex = pellexFromTextState(*plxs);
      ASSERT(0 != pellex); // shouldn't be called with something other than special text state
      UINT cbCmp = 3 + pellex->cb; // length of end tag
      ASSERT(cbCmp < sizeof szEnd);
      _tcscpy(szEnd, _T("</"));
      _tcscat(szEnd, pellex->sz);
      _tcscat(szEnd, _T(">"));
  
      while (cbCur < cbLen)
      {
          if (_T('<') == pchLine[cbCur])
          {
              if ((cbLen - cbCur >= cbCmp) && (0 == _tcsnicmp(szEnd, &pchLine[cbCur], cbCmp)))
              {
                  *plxs &= ~TEXTMASK; // special text modes are exclusive
                  token.ibTokMac = cbCur;
                  return cbCur;
              }
              else if ((cbCur + 1 < cbLen) && (_T('%') == pchLine[cbCur+1]))
              {
                  *plxs |= inHTXTag;
                  token.ibTokMac = cbCur;
                  break;
              }
              else
                  cbCur++;
          }
          else
              cbCur += _tclen(&pchLine[cbCur]);
      }
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  ////////////////////////////////////////////////////////////////////////////
  
  BOOL MakeSublang(PSUBLANG ps, UINT id, const TCHAR *strName, UINT nIdTemplate, CLSID clsid)
  {
      int len;
  
      ASSERT( NULL != ps );
  
      ps->szSubLang = NULL;
      ps->lxsInitial = LxsFromSubLangIndex(id);
      ps->nIdTemplate = nIdTemplate;
      ps->clsidTemplate = clsid;
  
      if ((len = lstrlen(strName)) != 0)
      {
          LPTSTR szNew = new TCHAR [len+1];
          if (NULL != szNew)
          {
              _tcscpy(szNew,strName);
              ps->szSubLang = szNew;
              return TRUE;
          }
      }
      return FALSE;
  }
  
  // Set sublang and tableset array members,
  // putting the default one in 0th position.
  //
  void SetLanguage(TCHAR * strDefault, PSUBLANG rgSublang,
                   PTABLESET pTab, UINT & index, UINT nIdTemplate, CLSID clsid)
  {
      if (pTab != NULL)
      {
          int i;
          if (lstrcmp(strDefault, pTab->Name()) == 0)
              i = 0;
          else
              i = index;
          if (MakeSublang(rgSublang+i, i, pTab->Name(), nIdTemplate, clsid))
          {
              g_arpTables[i] = pTab;
              if (i)
                  index++;
              else
                  g_pTable = pTab;
          }
          else
              delete pTab;
      }
  }
  
  CTableSet * FindTable(CTableSet ** rgpts, TCHAR *strName)
  {
      for (int n = 0; rgpts[n]; n++)
      {
          if (rgpts[n]->Name() == strName)
          //if (strcmp(rgpts[n]->Name(), strName) == 0)
              return rgpts[n];
      }
      return NULL;
  }
  
  CTableSet * FindTable(CTableSet ** rgpts, CTableSet * pts)
  {
      for (int n = 0; rgpts[n]; n++)
      {
          if (rgpts[n] == pts)
              return rgpts[n];
      }
      return NULL;
  }
  
  void RemoveTable(CTableSet ** rgpts, CTableSet *pts)
  {
      int n;
      for (n = 0; rgpts[n]; n++)
      {
          if (rgpts[n] == pts)
          {
              for(; rgpts[n]; n++)
                  rgpts[n] = rgpts[n+1];
              return;
          }
      }
  }
  
  CTableSet * MakeTableSet(CTableSet ** /*rgpts*/, RWATT_T att, UINT nIdName)
  {
      return new CStaticTableSet(att, nIdName);
  }
  
  ////////////////////////////////////////////////////////////////////////
  // GetToken()
  //
  UINT GetToken(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      ASSERT (cbCur < cbLen);
      if(cbCur > cbLen)
          return cbCur;
  
      UINT cbCount = 0;
  
      // init token
      token.tok = 0;
  
      // initialize location where token starts
      token.ibTokMin = cbCur;
  
      if (*plxs & inHTXTag)
          cbCount = FindServerScript(pchLine, cbLen, cbCur, plxs, token);
      else if (*plxs & inSCRIPT && !(*plxs & inTag) && !(*plxs & inServerASP))
      {
          // NOTE that we want to skip tokenizing scripts that are special to triedit
          // when we wrap server-side scripts in client-side scripts, we set a dummy
          // language as 'serverasp'. inServerASP is set in that case.
          cbCount = FindClientScriptEnd(pchLine, cbLen, cbCur, plxs, token);
      }
      else if (*plxs & inComment)  // in a comment
      {
          if (*plxs & inSCRIPT)
              *plxs |= inScriptText;
          COMMENTTYPE ct = IfHackComment(pchLine, cbLen, cbCur, plxs, token);
          if (ct == CT_METADATA)
          {
              // Treat as an element
              cbCount = FindNextToken(pchLine, cbLen, cbCur, plxs, token);
              // Remove inBangTag
              *plxs &= ~inBangTag;
          }
          else if (ct == CT_IECOMMENT)
              cbCount = token.ibTokMac;
          else
              cbCount = FindEndComment(pchLine, cbLen, cbCur, plxs, token);
      }
      else if (*plxs & INSTRING)  // in a string
          cbCount = FindEndString(pchLine, cbLen, cbCur, plxs, token);
      else
          cbCount = FindNextToken(pchLine, cbLen, cbCur, plxs, token);
  
      token.tokClass = MapToken(token.tokClass, *plxs);
      return cbCount;
  }
  
  ///////////////////////////////////////////////////////////////////////////////////
  // IfHackComment
  //
  // Probe ahead in the current line to see if we have what IE recognizes
  // as the end of a comment ("->"). This does not conform to RFC 1866 or SGML,
  // but suppports browser behavior. This lets us tolerate comments of the
  // form: "<!--- whatever ->"
  // (note how it ends)
  //
  // Returns a COMMENTTYPE enum.
  //  0 if norma comment
  //  1 if IE comment
  //  -1 if METADATA comment
  //
  // Proper comments are scanned using FindEndComment().
  //
  COMMENTTYPE IfHackComment(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      token.tokClass = tokComment;
      while (cbCur+1 < cbLen)
      {
          if(_tcsnicmp(&pchLine[cbCur], _T("METADATA"), lstrlen(_T("METADATA"))) == 0)
          {
              token.ibTokMac = cbCur + 1; // include second dash??
              *plxs &= ~inComment;
              // Remove inBangTag
              *plxs &= ~inBangTag;
              *plxs |= inTag;
              return CT_METADATA; // METADATA
          }
          else if (pchLine[cbCur] == '-' && pchLine[cbCur + 1] == '>')
          {
              token.ibTokMac = cbCur + 1;
              *plxs &= ~inComment;
              *plxs &= ~inScriptText;
              return CT_IECOMMENT;
          }
          else
          {
              cbCur += _tclen(&pchLine[cbCur]);
          }
      }
      return CT_NORMAL;
  }
  
  
  UINT FindServerScript(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      int cb;
  
      // parse HTX start tag
      if (*pCurrent == _T('<') && (cbCur+1 < cbLen) && *(pCurrent+1) == '%')
      {
          token.tokClass = tokTag;
          token.tok = TokTag_SSSOPEN;
          token.ibTokMac = cbCur + 2;
          *plxs |= inHTXTag;
          return token.ibTokMac;
      }
  
      ASSERT(*plxs & inHTXTag); // should be in HTXTag state here
  
      if (*pCurrent == _T('%') && (cbCur+1 < cbLen) && *(pCurrent+1) == '>')
      {
          token.tok = TokTag_SSSCLOSE;
          token.tokClass = tokSSS; //tokTag;
          token.ibTokMac = cbCur + 2;
          *plxs &= ~inHTXTag;
          if (*plxs & inNestedQuoteinSSS)
              *plxs &= ~inNestedQuoteinSSS;
          return token.ibTokMac;
      }
  
      token.tokClass = tokSSS;
  
      while (cbCur < cbLen)
      {
          if (*pCurrent == _T('%') && (cbCur+1 < cbLen) && (*(pCurrent+1) == _T('>')))
              break;
          if (   *pCurrent == _T('"') 
              && *plxs&inTag
              && *plxs&inHTXTag
              && *plxs&inAttribute
              && *plxs&inString
              )
              *plxs |= inNestedQuoteinSSS;
  
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
  
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  ///////////////////////////////////////////////////////////////////////////////////
  // FindClientScriptEnd()
  //
  // HTMED CHANGE: Find the end of client script block
  //
  UINT FindClientScriptEnd(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      int cb;
  
      TCHAR rgEndScript[] = _T("</SCRIPT");
      int cchEndScript = (wcslen(rgEndScript) - 1);
  
      if( cbCur + cchEndScript < cbLen &&
          0 == _tcsnicmp(pCurrent, rgEndScript, cchEndScript))
      {
          token.tokClass = tokTag;
          token.tok = TokTag_END;
          *plxs &= ~inSCRIPT;
          *plxs |= inEndTag;
          token.ibTokMac = cbCur + 2;
          return token.ibTokMac;
      }
  
      token.tokClass = tokSpace;
  
      while (cbCur < cbLen)
      {
          if (*pCurrent == _T('<') && (cbCur+1 < cbLen) && (*(pCurrent+1) == _T('/')))
          {
              // Check if found end </SCRIPT
              if( cbCur + cchEndScript < cbLen &&
                  0 == _tcsnicmp(pCurrent, rgEndScript, cchEndScript))
              {
                  // Check if found end </SCRIPT
                  break;
              }
          }
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
  
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  ///////////////////////////////////////////////////////////////////////////////////
  // FindEndComment()
  //
  // Find the end of comment ("--").
  //
  UINT FindEndComment(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      BOOL bEndComment = FALSE;
      int cb;
  
      ASSERT(*plxs & inComment); // must be in a comment now
  
      token.tokClass = tokComment;
  
      while (!bEndComment && cbCur < cbLen)
      {
          if (*pCurrent == _T('-'))  // check the character to see if it's the first "-" in "--"
          {
              pCurrent++;
              cbCur++;
              if ((cbCur < cbLen) &&
                  (*pCurrent == _T('-'))) // we're possibly at the end, so search for the final "--" pair
              {
                  bEndComment = TRUE;
              }
          }
          else
          {
              cb = _tclen(pCurrent);
              cbCur += cb;
              pCurrent += cb;
          }
      }
      if (cbCur < cbLen)
      {
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
  
      token.ibTokMac = cbCur;
  
      // reset state if we reach end of comment
      if (bEndComment)
          *plxs &= ~inComment;
  
      return cbCur;
  }
  
  /////////////////////////////////////////////////////////////
  // FindEndEntity()
  //
  // Find the end of the special character sequence (ends with ; or whitespace).
  //
  UINT FindEndEntity(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * /*plxs*/, TXTB & token)
  {
      token.tokClass = tokEntity;
      int cb = GetTokenLength(pchLine, cbLen, cbCur);
      if (pchLine[cbCur + cb] == ';')
          cb++;
      token.ibTokMac = cbCur + cb;
      return token.ibTokMac;
  }
  
  /////////////////////////////////////////////////////////////
  // Find an entity reference or non-entity ref, literal "&..."
  //
  UINT FindEntityRef(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * /*plxs*/, TXTB & token)
  {
      ASSERT(cbCur < cbLen);
      ASSERT(pchLine[cbCur] == '&'); // must be on ERO
  
      cbCur++;
      if (cbCur == cbLen)
      {
  NotEntity:
          token.tokClass = tokIDENTIFIER; // plain text
          token.ibTokMac = cbCur;
          return cbCur;
      }
  
      if (pchLine[cbCur] == '#')
      {
          // parse and check valid number
          if (!IsNumber(pchLine, cbLen, cbCur + 1, token))
              goto NotEntity;
  
          // must be <= 3 digits
          if (token.ibTokMac - (cbCur + 1) > 3)
              goto NotEntity;
  
          // validate range
          TCHAR szNum[4];
          _tcsncpy(szNum, &pchLine[cbCur + 1], 3);
          if (_tcstoul(szNum, 0, 10) > 255)
              goto NotEntity;
  
          // we now have a valid numeric entity ref
  
          token.tokClass = tokEntity;
          cbCur = token.ibTokMac;
  
          // scan for end of entity ref
  
          // scan rest of alphanumeric token
          // REVIEW: Is this correct? IE 4.40.308 behaves this way
          while ((cbCur < cbLen) && IsCharAlphaNumeric(pchLine[cbCur]))
              cbCur++;
  
          // scan delimiter
          if (cbCur < cbLen)
              cbCur++;
          token.ibTokMac = cbCur;
          return cbCur;
      }
      else if (!IsCharAlpha(pchLine[cbCur]))
      {
          goto NotEntity;
      }
      else
      {
          // parse and check entity name
          UINT nLen = GetTokenLength(pchLine, cbLen, cbCur);
          if (!g_pTable->FindEntity(&pchLine[cbCur], nLen))
              goto NotEntity;
  
          cbCur += nLen;
          // eat delimiter if necessary
          if ((cbCur < cbLen) &&
              (pchLine[cbCur] == ';' || IsWhiteSpace(pchLine[cbCur])))
              cbCur++;
          token.tokClass = tokEntity;
          token.ibTokMac = cbCur;
          return cbCur;
      }
  }
  
  
  /////////////////////////////////////////////////////////////
  // FindEndValue
  // Find the end of an unquoted value.
  //
  // Scan for whitespace or end if tag
  //
  UINT FindValue(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      ASSERT(cbCur < cbLen);
  
      do
      {
          cbCur++;
      } while ( cbCur < cbLen &&
          !IsWhiteSpace(pchLine[cbCur]) &&
          pchLine[cbCur] != '>' );
  
      token.tokClass = tokValue;
      token.ibTokMac = cbCur;
  
      // switch from value to attribute
      *plxs &= ~inValue;
      *plxs |= inAttribute;
  
      return cbCur;
  }
  
  /////////////////////////////////////////////////////////////
  // FindEndString()
  // Find the end of the string.
  // Should only be called when we are in the string mode already.
  //
  UINT FindEndString (LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      int cb;
      BOOL bInString = TRUE;
      TCHAR chDelim;
  
      ASSERT (*plxs & INSTRING); // must be in a string now
  
      token.tokClass = tokString;
      chDelim = (*plxs & inStringA) ? _T('\'') : _T('"');
  
      while (bInString && cbCur < cbLen)
      {
          if (*pCurrent == chDelim)
          {
              *plxs &= ~INSTRING;
              bInString = FALSE;
              SetValueSeen(plxs);
          }
          else if (*pCurrent == _T('<') &&
              cbCur+1 < cbLen &&
              *(pCurrent+1) == _T('%'))
          {
              *plxs |= inHTXTag;
              break;
          }
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  //////////////////////////////////////////////////////////////////
  //
  UINT FindTagOpen(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      ASSERT(pchLine[cbCur] == '<');
      token.tokClass = tokTag;
      *plxs &= ~inScriptText;     // turn off script coloring when inside tags
      cbCur++;
  
      if (cbCur == cbLen)
      {
          *plxs |= inTag;
      }
      else
      {
  #ifdef NEEDED // copied from htmed\lexer.cpp
          //
          // HTMED CHANGE:
          // REVIEW(cgomes): Figure out if I should turn off inSCRIPT in any of the
          // following cases.  Right now I only do it for the </ case.
          //
  #endif //NEEDED         
          switch (pchLine[cbCur])
          {
          case '!': // MDO - Markup Declaration Open
              cbCur++;
              *plxs |= inBangTag;
              token.tok = TokTag_BANG;
              break;
  
          case '/': // End tag
              cbCur++;
              *plxs |= inEndTag;
              token.tok = TokTag_END;
  #ifdef NEEDED // copied from htmed\lexer.cpp
              // HTMED CHANGE:
              // REVIEW(cgomes): Colorizer bug: it never removes the inSCRIPT state
              //  This removes the inSCRIPT in the case <SCRIPT <BODY>
              //  in this case <BODY is in error.
              //
              *plxs &= ~inSCRIPT;
  #endif //NEEDED         
              break;
  
          // REVIEW: PI is SGML -- not in HTML, but might be added
          case '?': // PI - Processing Instruction
              cbCur++;
              *plxs |= inPITag;
              token.tok = TokTag_PI;
              break;
  
          case '%': // HTX -- ODBC server HTML extension
              cbCur++;
              *plxs |= inHTXTag;
              token.tok = TokTag_SSSOPEN;
              break;
  
          default: // Tag
              if (IsCharAlpha(pchLine[cbCur]))
              {
                  *plxs |= inTag;
                  token.tok = TokTag_START;
              }
              else
                  token.tokClass = tokIDENTIFIER; // NOT a TAG
              break;
          }
      }
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  //////////////////////////////////////////////////////////////////
  //  FindText
  //  Scan a token of text
  //      NOTE DO NOT MODIFY this function, mainly b/c the side effects
  //              will be hard to find, and will break the way
  //              that everything works.
  //
  UINT FindText(LPCTSTR pchLine, UINT cbLen, UINT cbCur, TXTB & token)
  {
      //BOOL fExtraSpace = FALSE;
      //int cSpace = 0;
  
      ASSERT (cbCur < cbLen);
  
      token.tokClass = tokIDENTIFIER;
  
      //if (pchLine[cbCur] == ' ' && !fExtraSpace)
      //  fExtraSpace = TRUE;
      cbCur += _tclen(&pchLine[cbCur]);
      while (cbCur < cbLen)
      {
          switch (pchLine[cbCur])
          {
          case _T('\0'):
          case _T('\n'):
          case _T('<'):
          case _T('&'):
              //if (cSpace > 0) // found extra spaces so remember them somewhere
              goto ret;
              break;
          //case _T(' '):
          //  if (!fExtraSpace)
          //      fExtraSpace = TRUE;
          //  else
          //      cSpace++;
          //  break;
          default:
              //if (cSpace > 0) // found extra spaces so remember them somewhere
              //cSpace = 0;
              //fExtraSpace = FALSE;
              break;
          }
          cbCur += _tclen(&pchLine[cbCur]);
      }
  
  ret:
      token.ibTokMac = cbCur;
      return cbCur;
  }
  
  //////////////////////////////////////////////////////////////////
  // FindNextToken()
  //  Find the next token in the line
  //
  UINT FindNextToken(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      ASSERT (cbCur < cbLen);
      HINT hint;
  
      if (!(*plxs & INTAG)) // scanning text
      {
          if (*plxs & TEXTMASK)
          {
              if (*plxs & inCOMMENT)
                  token.tokClass = tokComment;
              else
                  token.tokClass = tokIDENTIFIER;
              // probe for end tag </comment>
              UINT cbEnd = FindEndTag(pchLine, cbLen, cbCur, plxs, token);
              if (cbEnd > cbCur) // parsed a nonzero-length token
              {
                  return cbEnd;
              }
              //else fall through to normal processing
          }
          hint = GetTextHint(pchLine, cbLen, cbCur, plxs, token);
          switch (hint)
          {
          case HTA:
              // begin a tag
              return FindTagOpen(pchLine, cbLen, cbCur, plxs, token);
  
          case HEN:
              // scan an entity reference
              token.ibTokMac = FindEntityRef(pchLine, cbLen, cbCur, plxs, token);
              return token.ibTokMac;
  
          case EOS:
          case ONL:
              return token.ibTokMac;
  
          case ERR:
          default:
              // scan text as a single token
              // If the editor uses token info for more than coloring
              //   (e.g. extended selections), then this will need to
              //   return smaller chunks.
              if (*plxs & inSCRIPT)
                  *plxs |= inScriptText;
              return FindText(pchLine, cbLen, cbCur, token);
              break;
          }
  
          return cbCur;
      }
  
      ASSERT(*plxs & INTAG); // must be in a tag here
  
      BOOL bError = FALSE;
      hint = GetHint(pchLine, cbLen, cbCur, plxs, token);
      switch (hint)
      {
      case HTE:
          // Tag end: remove all tag state bits
          *plxs &= ~TAGMASK;
          cbCur++;
          token.tokClass = tokTag;
          token.tok = TokTag_CLOSE;
          token.ibTokMac = cbCur;
          break;
  
      case HNU:
  #if 0  // lexing HTML instance, not a DTD!
          if (!IsNumber(pchLine, cbLen, cbCur, token))
              bError = TRUE;
          if (SetValueSeen(plxs))
              token.tokClass = tokValue;
          break;
  #else
          // fall through
  #endif
  
      case HRN: // reserved name start: #
  #if 1  // lexing HTML instance, not a DTD!
          // simple nonwhitespace stream
          if (!(*plxs & inValue))
              bError = TRUE;
          FindValue(pchLine, cbLen, cbCur, plxs, token);
          if (bError)
          {
              token.tokClass = tokSpace;
              bError = FALSE; //"corrected" the error
          }
  #else
          cbCur++;
          if (cbCur == cbLen)
              token.tokClass = tokOp;
          else
          {
              if (IsIdChar(pchLine[cbCur]))
              {
                  cbCur++;
                  while (cbCur < cbLen && IsIdChar(pchLine[cbCur]))
                      cbCur++;
                  token.tokClass = tokResName;
              }
              else
                  token.tokClass = tokOp;
          }
          token.ibTokMac = cbCur;
          if (SetValueSeen(plxs))
              token.tokClass = tokValue;
  #endif
          break;
  
      case HEP: // parameter entity: %
  #if 1  // lexing HTML instance, not a DTD!
          goto BadChar;
  #else
          cbCur++;
          if (cbCur == cbLen)
          {
              token.tokClass = tokOp;
              token.ibTokMac = cbCur;
          }
          else
          {
              if (IsIdChar(pchLine[cbCur]))
              {
                  token.ibTokMac = FindEndEntity(pchLine, cbLen, cbCur, plxs, token);
                  token.tokClass = tokParEnt;
              }
              else
              {
                  token.ibTokMac = cbCur;
                  token.tokClass = tokOp;
              }
          }
          if (SetValueSeen(plxs))
              token.tokClass = tokValue;
  #endif
          break;
  
      // ported HTMED change (walts) -- handle some chars as valid start char for attribute values.
      case HAV:
          {
          if (!(*plxs & inTag) || !SetValueSeen(plxs))
              goto BadChar;   // not in tag or attribute value.
  
          int iTokenLength = GetValueTokenLength(pchLine, cbLen, cbCur);
          token.ibTokMac = token.ibTokMin + iTokenLength;
          token.tokClass = tokValue;
          break;
          }
      // ported HTMED change (walts) -- handle some chars as valid start char for attribute values.
  
      case HKW:  // identifier
          {
              int iTokenLength = GetTokenLength(pchLine, cbLen, cbCur);
              token.ibTokMac = token.ibTokMin + iTokenLength;
              token.tokClass = tokName;
              //FUTURE: Don't scan attributes in an end tag
              if (*plxs & (inTag|inEndTag))
              {
                  if (*plxs & inAttribute)
                  {
                      IsAttributeName(pchLine, cbCur, iTokenLength, token);
                      // don't change attribute/value state here
                      // we only look for values after we've seen "=" in case OEQ below
  
                      // REVIEW(cgomes): what if more attributes follow
                      // the SPAN??
                      // if found STARTSPAN then pretend I am not in a tag
                      if(token.tok == TokAttrib_STARTSPAN)
                          *plxs &= ~(inTag | inAttribute);
                      // if found ENDSPAN then goback to comment state
                      else if(token.tok == TokAttrib_ENDSPAN)
                      {
                          *plxs &= ~(inTag | inAttribute);
                          *plxs |= inBangTag | inComment;
                      }
                  }
                  else if (SetValueSeen(plxs))
                  {
                      // REVIEW (walts)
                      // Handle the client side script language detection here for the
                      // following case (language attribute value is NOT wrapped by quotes.)
                      // <SCRIPT LANGUAGE=VBScript>
                      if (*plxs & inSCRIPT)
                      {
                          SetScriptLanguage(&pchLine[cbCur], plxs);
                      }
  
                      //
                      // REVIEW(cgomes): It seems that any non-white space character
                      //      is valid for non-quoted attribute values.
                      //      Problem is that GetTokenLength is used to determine
                      //      the token length, which works great non-values,
                      //      but sucks egss for values.
                      //      I use GetValueTokenLength here to get the length
                      //      of value token.  GetValueTokenLength will not
                      //      stop till it hits a white space character.
                      //
  
                      iTokenLength = GetValueTokenLength(pchLine, cbLen, cbCur);
                      token.ibTokMac = token.ibTokMin + iTokenLength;
                      token.tokClass = tokName;
  
                      token.tokClass = tokValue;
                  }
                  else
                  {
                      IsElementName(pchLine, cbCur, iTokenLength, token);
                      // look for attributes
                      *plxs |= inAttribute;
                      // set content state
                      if (*plxs & inTag)
                          *plxs |= TextStateFromElement(&pchLine[token.ibTokMin], iTokenLength);
                      else if ((*plxs & inEndTag) && (*plxs & TEXTMASK))
                          *plxs &= ~TextStateFromElement(&pchLine[token.ibTokMin], iTokenLength);
                      else if ((*plxs & inEndTag) && (*plxs & inSCRIPT))
                          *plxs &= ~(inSCRIPT | inScriptText | inServerASP/* | inVBScript | inJavaScript*/);
                  }
              }
              else if (*plxs & inBangTag)
              {
                  // FUTURE: other <!...> items like "HTML", "PUBLIC"? -- nice for DTDs
                  //   Use a RW table for it if we do
  
                  // recognize <!DOCTYPE ...>  as 'element'
                  if ((iTokenLength == 7) &&
                      (0 == _tcsnicmp(&pchLine[cbCur], _T("doctype"), 7)))
                      token.tokClass = tokElem;
              }
              break;
          }
  
      case HST:  // string "..."
          *plxs |= inString;
          goto String;
  
      case HSL:  // string alternate '...'
          *plxs |= inStringA;
  String:
          cbCur++;
          token.ibTokMac = FindEndString(pchLine, cbLen, cbCur, plxs, token);
          SetValueSeen(plxs);
          // Handle the client side script language detection here for the
          // following case (language attribute value is wrapped by quotes.)
          // <SCRIPT LANGUAGE="VBScript">
          if((*plxs & inSCRIPT) && (*plxs & inAttribute))
          {
              SetScriptLanguage(&pchLine[cbCur], plxs);
          }
          break;
  
      case HWS: // tag whitespace
          do
          {
              cbCur++;
          } while (cbCur < cbLen && IsWhiteSpace(pchLine[cbCur]));
          token.tokClass = tokSpace;
          token.ibTokMac = cbCur;
          break;
  
      case OEQ:
          // GetHint has set token info
          if (*plxs & inAttribute)
          {
              // start looking for values
              *plxs &= ~inAttribute;
              *plxs |= inValue;
          }
          else
              goto BadChar;
          break;
  
      case HTA:
          if (cbCur+1 < cbLen && '%' == pchLine[cbCur+1])
          {
              SetValueSeen(plxs);
              return FindTagOpen(pchLine, cbLen, cbCur, plxs, token);
          }
          // else fall through
      case ERR:
      case HEN:
  BadChar:
          token.tokClass = tokSpace;
  
          // DS96# 10116 [CFlaat]: we can be in DBCS here, and so we need
          //     to make sure that our increment is double-byte aware
          cbCur += _tcsnbcnt(pchLine + cbCur, 1); // byte count for current char
          ASSERT(cbCur <= cbLen);
          token.ibTokMac = cbCur;
          break;
  
      // ported HTMED CHANGE (walts) - added this case to handle dbcs attribute values.
      case HDB:
          {
          // DBCS char.  Handle for attribute values within tag.
          if (!SetValueSeen(plxs))
              goto BadChar;
  
          int iTokenLength = GetValueTokenLength(pchLine, cbLen, cbCur);
          token.ibTokMac = token.ibTokMin + iTokenLength;
          token.tokClass = tokValue;
          }
          break;
      // ported HTMED CHANGE END
  
      default:
          // GetHint has set token info
          if (token.tokClass != tokComment && (*plxs & inValue))
              FindValue(pchLine, cbLen, cbCur, plxs, token);
          break;
      }
      if (bError)
          IsUnknownID(pchLine, cbLen, cbCur, token);
      return token.ibTokMac;
  }
  
  ////////////////////////////////////////////////////////////////////
  // GetTextHint()
  // Like GetHint when scanning text -- look only for tags and entities
  //
  HINT GetTextHint(LPCTSTR pchLine, UINT /*cbLen*/, UINT cbCur, DWORD * /*plxs*/, TXTB & token)
  {
      // if the character is bigger than 128 (dbcs) then return error
      if (pchLine[cbCur] & ~0x7F)
          return HDB;
  
      HINT hint = g_hintTable[pchLine[cbCur]];
  
      if (IsSingleOp(hint))
      {
          hint = ERR;
      }
      else if (hint == ONL || hint == EOS)
      {
          token.tokClass = tokOp;
          token.ibTokMac = cbCur + 1;
      }
      return hint;
  }
  
  ////////////////////////////////////////////////////////////////////
  // GetHint()
  //      Use hint table to guess what the next token going to be
  //      If it is a single operator, it will fill in the token info
  //      as well
  //
  HINT GetHint(LPCTSTR pchLine, UINT cbLen, UINT cbCur, DWORD * plxs, TXTB & token)
  {
      // if the character is bigger than 128 (dbcs) then return error
      if (pchLine[cbCur] & ~0x7F)
          return HDB;
  
      HINT hint = g_hintTable[pchLine[cbCur]];
  
      // check if it is a single op, new line or end of stream
      if (IsSingleOp(hint) || hint == ONL || hint == EOS)
      {
          token.tokClass = hint;
          token.ibTokMac = cbCur + 1;
      }
      else if (hint == ODA)
      {
          if ((cbCur + 1 < cbLen) &&
              (g_hintTable[pchLine[cbCur + 1]] == ODA) &&
              (*plxs & inBangTag))
          {
              cbCur += 2;
              *plxs |= inComment;
              COMMENTTYPE ct = IfHackComment(pchLine, cbLen, cbCur, plxs, token);
              if (ct == 0)
              {
                  token.tokClass = tokComment;
                  token.ibTokMac = cbCur;
              }
              else if(ct == CT_METADATA)
                  hint = HTA; // tag open
          }
          else
          {
              // single -
              token.tokClass = tokOp;
              token.ibTokMac = cbCur + 1;
          }
      }
      return hint;
  }
  
  ///////////////////////////////////////////////////////////////////
  // GetTokenLength ()
  //  return the length of a token identifier/keyword
  //
  UINT GetTokenLength(LPCTSTR pchLine, UINT cbLen, UINT cbCur)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      UINT cb;
      UINT cbOld = cbCur;
  
      if (IsCharAlphaNumeric(*pCurrent))
      {
          while (cbCur < cbLen && IsIdChar(*pCurrent))
          {
              cb = _tclen(pCurrent);
              cbCur += cb;
              pCurrent += cb;
          }
      }
      return (int) max((cbCur - cbOld), 1);
  }
  
  /*
  
      UINT GetValueTokenLength
  
      Description:
          Gets the length of the token.
          This version will accept any non whitespace character
          in the token.
  
  */
  UINT GetValueTokenLength(LPCTSTR pchLine, UINT cbLen, UINT cbCur)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      UINT cb;
      UINT cbOld = cbCur;
  
      while (cbCur < cbLen && !_istspace(*pCurrent) && IsValueChar(*pCurrent))
      {
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
      return (int) max((cbCur - cbOld), 1);
  }
  
  
  ////////////////////////////////////////////////////////////////
  // IsElementName ()
  //  lookup the keyword table to determine if it is a keyword or not
  //
  BOOL IsElementName(LPCTSTR pchLine, UINT cbCur, int iTokenLength, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      int iFound = NOT_FOUND;
  
      if (NOT_FOUND != (iFound = g_pTable->FindElement(pCurrent, iTokenLength)))
      {
          token.tokClass = tokElem;
          token.ibTokMac = cbCur + iTokenLength;
          token.tok = iFound; // set token
      }
      return (iFound != NOT_FOUND);
  }
  
  int IndexFromElementName(LPCTSTR pszName)
  {
      return g_pTable->FindElement(pszName, lstrlen(pszName));
  }
  
  ////////////////////////////////////////////////////////////////
  // IsAttributeName ()
  // lookup the keyword table to determine if it is a keyword or not
  //
  BOOL IsAttributeName(LPCTSTR pchLine, UINT cbCur, int iTokenLength, TXTB & token)
  {
      LPCTSTR pCurrent = &pchLine[cbCur];
      int iFound = NOT_FOUND;
  
      if (NOT_FOUND != (iFound = g_pTable->FindAttribute(pCurrent, iTokenLength)))
      {
          token.tokClass = tokAttr;
          // ENDSPAN__ is needed b/c the lexer does not recognize the
          // endspan-- as 2 seperate tokens.
          if(iFound == TokAttrib_ENDSPAN__)
          {
              // endspan-- found.  return TokAttrib_ENDSPAN
              // set ibTokMac to not include --.
              token.tok = TokAttrib_ENDSPAN;
              token.ibTokMac = cbCur + iTokenLength - 2;
          }
          else
          {
              token.ibTokMac = cbCur + iTokenLength;
              token.tok =  iFound; // set token
          }
      }
      return (iFound != NOT_FOUND);
  }
  
  //////////////////////////////////////////////////////////////////////////
  // IsIdentifier()
  // check if it is an identifier
  //
  BOOL IsIdentifier (int iTokenLength, TXTB & token)
  {
      if (iTokenLength > 0)
      {
          token.tokClass = tokName;
          token.ibTokMac = token.ibTokMin + iTokenLength;
          return TRUE;
      }
      else
          return FALSE;
  }
  
  ////////////////////////////////////////////////////////////////////
  // IsUnknownID ()
  //  Mark the next token as an ID
  //
  BOOL IsUnknownID (LPCTSTR pchLine, UINT cbLen, UINT cbCur, TXTB & token)
  {
      ASSERT(cbCur < cbLen);
      UINT cb;
      LPCTSTR pCurrent = &pchLine[cbCur];
  
      cb = _tclen(pCurrent);
      cbCur += cb;
      pCurrent += cb;
  
      while ((cbCur < cbLen) && IsIdChar(*pCurrent))
      {
          cb = _tclen(pCurrent);
          cbCur += cb;
          pCurrent += cb;
      }
  
      token.tokClass = tokSpace;
      token.ibTokMac = cbCur;
      return TRUE;
  }
  
  /////////////////////////////////////////////////////////////////////////
  // IsNumber()
  //  Check whether the next token is an SGML NUMTOKEN
  //
  BOOL IsNumber(LPCTSTR pchLine, UINT cbLen, UINT cbCur, TXTB & token)
  {
      if (cbCur >= cbLen)
          return FALSE;
  
      if (!_istdigit(pchLine[cbCur]))
          return FALSE;
  
      token.tokClass = tokNum;
  
      // assume all digits are one byte
      ASSERT(1 == _tclen(&pchLine[cbCur]));
      cbCur++;
  
      while (cbCur < cbLen && _istdigit(pchLine[cbCur]))
      {
          // assume all digits are one byte
          ASSERT(1 == _tclen(&pchLine[cbCur]));
          cbCur++;
      }
  
      token.ibTokMac = cbCur;
      return TRUE;
  }
  
  
  
  /* end of file */
  