context: /* --------------------------------------------------------------------------
   * Copyrights
   *
   * Portions created by or assigned to Cursive Systems, Inc. are
   * Copyright (c) 2002-2008 Cursive Systems, Inc.  All Rights Reserved.  Contact
   * information for Cursive Systems, Inc. is available at
   * http://www.cursive.net/.
   *
   * License
   *
   * Jabber-Net is licensed under the LGPL.
   * See LICENSE.txt for details.
   *
   * xpnet is a deriviative of James Clark's XP.  See copying.txt for more info.
   * --------------------------------------------------------------------------*/
  
  namespace xpnet
  {
      using bedrock.util;
  
      /// <summary>
      /// Tokens that might have been found
      /// </summary>
      [SVN(@"$Id$")]
      public enum TOK
      {
          /**
           * Represents one or more characters of data.
           */
          DATA_CHARS,
  
          /**
           * Represents a newline (CR, LF or CR followed by LF) in data.
           */
          DATA_NEWLINE,
  
          /**
           * Represents a complete start-tag <code>&lt;name&gt;</code>,
           * that doesn't have any attribute specifications.
           */
          START_TAG_NO_ATTS,
  
          /**
           * Represents a complete start-tag <code>&lt;name
           * att="val"&gt;</code>, that contains one or more
           * attribute specifications.
           */
          START_TAG_WITH_ATTS,
  
          /**
           * Represents an empty element tag <code>&lt;name/&gt;</code>,
           * that doesn't have any attribute specifications.
           */
          EMPTY_ELEMENT_NO_ATTS,
  
          /**
           * Represents an empty element tag <code>&lt;name
           * att="val"/&gt;</code>, that contains one or more
           * attribute specifications.
           */
          EMPTY_ELEMENT_WITH_ATTS,
  
          /**
           * Represents a complete end-tag <code>&lt;/name&gt;</code>.
           */
          END_TAG,
  
          /**
           * Represents the start of a CDATA section
           * <code>&lt;![CDATA[</code>.
           */
          CDATA_SECT_OPEN,
  
          /**
           * Represents the end of a CDATA section <code>]]&gt;</code>.
           */
          CDATA_SECT_CLOSE,
  
          /**
           * Represents a general entity reference.
           */
          ENTITY_REF,
  
          /**
           * Represents a general entity reference to a one of the 5
           * predefined entities <code>amp</code>, <code>lt</code>,
           * <code>gt</code>, <code>quot</code>, <code>apos</code>.
           */
          MAGIC_ENTITY_REF,
  
          /**
           * Represents a numeric character reference (decimal or
           * hexadecimal), when the referenced character is less
           * than or equal to 0xFFFF and so is represented by a
           * single char.
           */
          CHAR_REF,
  
          /**
           * Represents a numeric character reference (decimal or
           * hexadecimal), when the referenced character is greater
           * than 0xFFFF and so is represented by a pair of chars.
           */
          CHAR_PAIR_REF,
  
          /**
           * Represents a processing instruction.
           */
          PI,
  
          /**
           * Represents an XML declaration or text declaration (a
           * processing instruction whose target is
           * <code>xml</code>).
           */
          XML_DECL,
  
          /**
           * Represents a comment <code>&lt;!-- comment --&gt;</code>.
           * This can occur both in the prolog and in content.
           */
          COMMENT,
  
          /**
           * Represents a white space character in an attribute
           * value, excluding white space characters that are part
           * of line boundaries.
           */
          ATTRIBUTE_VALUE_S,
  
          /**
           * Represents a parameter entity reference in the prolog.
           */
          PARAM_ENTITY_REF,
  
          /**
           * Represents whitespace in the prolog.
           * The token contains one or more whitespace characters.
           */
          PROLOG_S,
  
          /**
           * Represents <code>&lt;!NAME</code> in the prolog.
           */
          DECL_OPEN,
  
          /**
           * Represents <code>&gt;</code> in the prolog.
           */
          DECL_CLOSE,
  
          /**
           * Represents a name in the prolog.
           */
          NAME,
  
          /**
           * Represents a name token in the prolog that is not a name.
           */
          NMTOKEN,
  
          /**
           * Represents <code>#NAME</code> in the prolog.
           */
          POUND_NAME,
  
          /**
           * Represents <code>|</code> in the prolog.
           */
          OR,
  
          /**
           * Represents a <code>%</code> in the prolog that does not start
           * a parameter entity reference.
           * This can occur in an entity declaration.
           */
          PERCENT,
  
          /**
           * Represents a <code>(</code> in the prolog.
           */
          OPEN_PAREN,
  
          /**
           * Represents a <code>)</code> in the prolog that is not
           * followed immediately by any of
           *  <code>*</code>, <code>+</code> or <code>?</code>.
           */
          CLOSE_PAREN,
  
          /**
           * Represents <code>[</code> in the prolog.
           */
          OPEN_BRACKET,
  
          /**
           * Represents <code>]</code> in the prolog.
           */
          CLOSE_BRACKET,
  
          /**
           * Represents a literal (EntityValue, AttValue, SystemLiteral or
           * PubidLiteral).
           */
          LITERAL,
  
          /**
           * Represents a name followed immediately by <code>?</code>.
           */
          NAME_QUESTION,
  
          /**
           * Represents a name followed immediately by <code>*</code>.
           */
          NAME_ASTERISK,
  
          /**
           * Represents a name followed immediately by <code>+</code>.
           */
          NAME_PLUS,
  
          /**
           * Represents <code>&lt;![</code> in the prolog.
           */
          COND_SECT_OPEN,
  
          /**
           * Represents <code>]]&gt;</code> in the prolog.
           */
          COND_SECT_CLOSE,
  
          /**
           * Represents <code>)?</code> in the prolog.
           */
          CLOSE_PAREN_QUESTION,
  
          /**
           * Represents <code>)*</code> in the prolog.
           */
          CLOSE_PAREN_ASTERISK,
  
          /**
           * Represents <code>)+</code> in the prolog.
           */
          CLOSE_PAREN_PLUS,
  
          /**
           * Represents <code>,</code> in the prolog.
           */
          COMMA,
      };
  
      /// <summary>
      /// Base tokenizer class
      /// </summary>
      [SVN(@"$Id$")]
      public abstract class Encoding
      {
          // Bytes with type < 0 may not be data in content.
          // The negation of the lead byte type gives the total number of bytes.
  
          /// <summary>
          /// Need more bytes
          /// </summary>
          protected const int BT_LEAD2 = -2;
  
          /// <summary>
          /// Need more bytes
          /// </summary>
          protected const int BT_LEAD3 = -3;
  
          /// <summary>
          /// Need more bytes
          /// </summary>
          protected const int BT_LEAD4 = -4;
  
          /// <summary>
          /// Not XML
          /// </summary>
          protected const int BT_NONXML = BT_LEAD4 - 1;
  
          /// <summary>
          /// Malformed XML
          /// </summary>
          protected const int BT_MALFORM = BT_NONXML - 1;
  
          /// <summary>
          /// Less than
          /// </summary>
          protected const int BT_LT = BT_MALFORM - 1;
  
          /// <summary>
          /// Ampersand
          /// </summary>
          protected const int BT_AMP = BT_LT - 1;
  
          /// <summary>
          /// right square bracket
          /// </summary>
          protected const int BT_RSQB = BT_AMP - 1;
  
          /// <summary>
          /// carriage return
          /// </summary>
          protected const int BT_CR = BT_RSQB - 1;
  
          /// <summary>
          /// line feed
          /// </summary>
          protected const int BT_LF = BT_CR - 1;
  
          // Bytes with type >= 0 are treated as data in content.
  
          /// <summary>
          /// greater than
          /// </summary>
          protected const int BT_GT = 0;
  
          /// <summary>
          /// Quote
          /// </summary>
          protected const int BT_QUOT = BT_GT + 1;
  
          /// <summary>
          /// Apostrophe
          /// </summary>
          protected const int BT_APOS = BT_QUOT + 1;
  
          /// <summary>
          /// Equal sign
          /// </summary>
          protected const int BT_EQUALS = BT_APOS + 1;
  
          /// <summary>
          /// Question mark
          /// </summary>
          protected const int BT_QUEST = BT_EQUALS + 1;
  
          /// <summary>
          /// Exclamation point
          /// </summary>
          protected const int BT_EXCL = BT_QUEST + 1;
  
          /// <summary>
          /// Solidus (/)
          /// </summary>
          protected const int BT_SOL = BT_EXCL + 1;
  
          /// <summary>
          /// Semicolon
          /// </summary>
          protected const int BT_SEMI = BT_SOL + 1;
  
          /// <summary>
          /// Hash
          /// </summary>
          protected const int BT_NUM = BT_SEMI + 1;
  
          /// <summary>
          /// Left square bracket
          /// </summary>
          protected const int BT_LSQB = BT_NUM + 1;
  
          /// <summary>
          /// space
          /// </summary>
          protected const int BT_S = BT_LSQB + 1;
  
          /// <summary>
          ///
          /// </summary>
          protected const int BT_NMSTRT = BT_S + 1;
  
          /// <summary>
          ///
          /// </summary>
          protected const int BT_NAME = BT_NMSTRT + 1;
  
          /// <summary>
          /// Minus
          /// </summary>
          protected const int BT_MINUS = BT_NAME + 1;
  
          /// <summary>
          /// Other
          /// </summary>
          protected const int BT_OTHER = BT_MINUS + 1;
  
          /// <summary>
          /// Percent
          /// </summary>
          protected const int BT_PERCNT = BT_OTHER + 1;
  
          /// <summary>
          /// Left paren
          /// </summary>
          protected const int BT_LPAR = BT_PERCNT + 1;
  
          /// <summary>
          /// Right paren
          /// </summary>
          protected const int BT_RPAR = BT_LPAR + 1;
  
          /// <summary>
          ///
          /// </summary>
          protected const int BT_AST = BT_RPAR + 1;
  
          /// <summary>
          /// +
          /// </summary>
          protected const int BT_PLUS = BT_AST + 1;
  
          /// <summary>
          /// ,
          /// </summary>
          protected const int BT_COMMA = BT_PLUS + 1;
  
          /// <summary>
          /// Pipe
          /// </summary>
          protected const int BT_VERBAR = BT_COMMA + 1;
  
          /// <summary>
          /// What syntax do each of the ASCII7 characters have?
          /// </summary>
          protected static readonly int[] asciiTypeTable = new int[]
          {
              /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x04 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x08 */ BT_NONXML, BT_S, BT_LF, BT_NONXML,
              /* 0x0C */ BT_NONXML, BT_CR, BT_NONXML, BT_NONXML,
              /* 0x10 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x14 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x18 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x1C */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML,
              /* 0x20 */ BT_S, BT_EXCL, BT_QUOT, BT_NUM,
              /* 0x24 */ BT_OTHER, BT_PERCNT, BT_AMP, BT_APOS,
              /* 0x28 */ BT_LPAR, BT_RPAR, BT_AST, BT_PLUS,
              /* 0x2C */ BT_COMMA, BT_MINUS, BT_NAME, BT_SOL,
              /* 0x30 */ BT_NAME, BT_NAME, BT_NAME, BT_NAME,
              /* 0x34 */ BT_NAME, BT_NAME, BT_NAME, BT_NAME,
              /* 0x38 */ BT_NAME, BT_NAME, BT_NMSTRT, BT_SEMI,
              /* 0x3C */ BT_LT, BT_EQUALS, BT_GT, BT_QUEST,
              /* 0x40 */ BT_OTHER, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x44 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x48 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x4C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x50 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x54 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x58 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_LSQB,
              /* 0x5C */ BT_OTHER, BT_RSQB, BT_OTHER, BT_NMSTRT,
              /* 0x60 */ BT_OTHER, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x64 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x68 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x6C */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x70 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x74 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_NMSTRT,
              /* 0x78 */ BT_NMSTRT, BT_NMSTRT, BT_NMSTRT, BT_OTHER,
              /* 0x7C */ BT_VERBAR, BT_OTHER, BT_OTHER, BT_OTHER,
          };
  
          /// <summary>
          ///
          /// </summary>
          /// <param name="sourceBuf"></param>
          /// <param name="sourceStart"></param>
          /// <param name="sourceEnd"></param>
          /// <param name="targetBuf"></param>
          /// <param name="targetStart"></param>
          /// <returns></returns>
          protected abstract int convert(byte[] sourceBuf,
              int sourceStart, int sourceEnd,
              char[] targetBuf, int targetStart);
  
  
          private static Encoding utf8Encoding;
  
          private const byte UTF8_ENCODING = 0;
          private const byte UTF16_LITTLE_ENDIAN_ENCODING = 1;
          private const byte UTF16_BIG_ENDIAN_ENCODING = 2;
          private const byte INTERNAL_ENCODING = 3;
          private const byte ISO8859_1_ENCODING = 4;
          private const byte ASCII_ENCODING = 5;
  
          private static Encoding getEncoding(byte enc)
          {
              switch (enc)
              {
                  case UTF8_ENCODING:
                      if (utf8Encoding == null)
                          utf8Encoding = new UTF8Encoding();
                      return utf8Encoding;
                  /*
                  case UTF16_LITTLE_ENDIAN_ENCODING:
                      if (utf16LittleEndianEncoding == null)
                          utf16LittleEndianEncoding = new UTF16LittleEndianEncoding();
                      return utf16LittleEndianEncoding;
                  case UTF16_BIG_ENDIAN_ENCODING:
                      if (utf16BigEndianEncoding == null)
                          utf16BigEndianEncoding = new UTF16BigEndianEncoding();
                      return utf16BigEndianEncoding;
                  case INTERNAL_ENCODING:
                      if (internalEncoding == null)
                          internalEncoding = new InternalEncoding();
                      return internalEncoding;
                  case ISO8859_1_ENCODING:
                      if (iso8859_1Encoding == null)
                          iso8859_1Encoding = new ISO8859_1Encoding();
                      return iso8859_1Encoding;
                  case ASCII_ENCODING:
                      if (asciiEncoding == null)
                          asciiEncoding = new ASCIIEncoding();
                      return asciiEncoding;
                      */
              }
              return null;
          }
  
          private int minBPC;
  
          /// <summary>
          /// Constructor called by subclasses to set the minimum bytes per character
          /// </summary>
          /// <param name="minBPC"></param>
          protected Encoding(int minBPC)
          {
              this.minBPC = minBPC;
          }
  
          /// <summary>
          /// Get the byte type of the next byte.  There are guaranteed to be minBPC available bytes starting at off.
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <returns></returns>
          protected abstract int byteType(byte[] buf, int off);
  
          /// <summary>
          /// Really only works for ASCII7.
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <returns></returns>
          protected abstract char byteToAscii(byte[] buf, int off);
  
          /// <summary>
          /// This must only be called when c is an (XML significant)
          /// ASCII character.
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <param name="c"></param>
          /// <returns></returns>
          protected abstract bool charMatches(byte[] buf, int off, char c);
  
          /// <summary>
          /// Called only when byteType(buf, off) == BT_LEAD2
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <returns></returns>
          protected virtual int byteType2(byte[] buf, int off)
          {
              return BT_OTHER;
          }
  
          /// <summary>
          /// Called only when byteType(buf, off) == BT_LEAD3
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <returns></returns>
          private int byteType3(byte[] buf, int off)
          {
              return BT_OTHER;
          }
  
          /// <summary>
          /// Called only when byteType(buf, off) == BT_LEAD4
          /// </summary>
          /// <param name="buf"></param>
          /// <param name="off"></param>
          /// <returns></returns>
          private int byteType4(byte[] buf, int off)
          {
              return BT_OTHER;
          }
  
          private void check2(byte[] buf, int off)
          {
          }
  
          private void check3(byte[] buf, int off)
          {
          }
  
          private void check4(byte[] buf, int off)
          {
          }
  
          /**
           * Moves a position forward.  On entry, <code>pos</code> gives
           * the position of the byte at index <code>off</code> in
           * <code>buf</code>.  On exit, it <code>pos</code> will give
           * the position of the byte at index <code>end</code>, which
           * must be greater than or equal to <code>off</code>.  The
           * bytes between <code>off</code> and <code>end</code> must
           * encode one or more complete characters.  A carriage return
           * followed by a line feed will be treated as a single line
           * delimiter provided that they are given to
           * <code>movePosition</code> together.
           */
          protected abstract void movePosition(byte[] buf, int off, int end, Position pos);
  
          private void checkCharMatches(byte[] buf, int off, char c)
          {
              if (!charMatches(buf, off, c))
                  throw new InvalidTokenException(off);
          }
  
          /* off points to character following "<!-" */
  
          private TOK scanComment(byte[] buf, int off, int end, Token token)
          {
              if (off != end)
              {
                  checkCharMatches(buf, off, '-');
                  off += minBPC;
                  while (off != end)
                  {
                      switch (byteType(buf, off))
                      {
                          case BT_LEAD2:
                              if (end - off < 2)
                                  throw new PartialCharException(off);
                              check2(buf, off);
                              off += 2;
                              break;
                          case BT_LEAD3:
                              if (end - off < 3)
                                  throw new PartialCharException(off);
                              check3(buf, off);
                              off += 3;
                              break;
                          case BT_LEAD4:
                              if (end - off < 4)
                                  throw new PartialCharException(off);
                              check4(buf, off);
                              off += 4;
                              break;
                          case BT_NONXML:
                          case BT_MALFORM:
                              throw new InvalidTokenException(off);
                          case BT_MINUS:
                              if ((off += minBPC) == end)
                                  throw new PartialTokenException();
                              if (charMatches(buf, off, '-'))
                              {
                                  if ((off += minBPC) == end)
                                      throw new PartialTokenException();
                                  checkCharMatches(buf, off, '>');
                                  token.TokenEnd = off + minBPC;
                                  return TOK.COMMENT;
                              }
                              break;
                          default:
                              off += minBPC;
                              break;
                      }
                  }
              }
              throw new PartialTokenException();
          }
  
          /* off points to character following "<!" */
  
          private TOK scanDecl(byte[] buf, int off, int end, Token token)
          {
              if (off == end)
                  throw new PartialTokenException();
              switch (byteType(buf, off))
              {
                  case BT_MINUS:
                      return scanComment(buf, off + minBPC, end, token);
                  case BT_LSQB:
                      token.TokenEnd = off + minBPC;
                      return TOK.COND_SECT_OPEN;
                  case BT_NMSTRT:
                      off += minBPC;
                      break;
                  default:
                      throw new InvalidTokenException(off);
              }
              while (off != end)
              {
                  switch (byteType(buf, off))
                  {
                      case BT_PERCNT:
                          if (off + minBPC == end)
                              throw new PartialTokenException();
                          /* don't allow <!ENTITY% foo "whatever"> */
                          switch (byteType(buf, off + minBPC))
                          {
                              case BT_S:
                              case BT_CR:
                              case BT_LF:
                              case BT_PERCNT:
                                  throw new InvalidTokenException(off);
                          }
                          /* fall through */
                          goto case BT_S;
                      case BT_S:
                      case BT_CR:
                      case BT_LF:
                          token.TokenEnd = off;
                          return TOK.DECL_OPEN;
                      case BT_NMSTRT:
                          off += minBPC;
                          break;
                      default:
                          throw new InvalidTokenException(off);
                  }
              }
              throw new PartialTokenException();
          }
  
          private bool targetIsXml(byte[] buf, int off, int end)
          {
              bool upper = false;
              if (end - off != minBPC*3)
                  return false;
              switch (byteToAscii(buf, off))
              {
                  case 'x':
                      break;
                  case 'X':
                      upper = true;
                      break;
                  default:
                      return false;
              }
              off += minBPC;
              switch (byteToAscii(buf, off))
              {
                  case 'm':
                      break;
                  case 'M':
                      upper = true;
                      break;
                  default:
                      return false;
              }
              off += minBPC;
              switch (byteToAscii(buf, off))
              {
                  case 'l':
                      break;
                  case 'L':
                      upper = true;
                      break;
                  default:
                      return false;
              }
              if (upper)
                  throw new InvalidTokenException(off, InvalidTokenException.XML_TARGET);
              return true;
          }
  
          /* off points to character following "<?" */
  
          private TOK scanPi(byte[] buf, int off, int end, Token token)
          {
              int target = off;
              if (off == end)
                  throw new PartialTokenException();
              switch (byteType(buf, off))
              {
                  case BT_NMSTRT:
                      off += minBPC;
                      break;
                  case BT_LEAD2:
                      if (end - off < 2)
                          throw new PartialCharException(off);
                      if (byteType2(buf, off) != BT_NMSTRT)
                          throw new InvalidTokenException(off);
                      off += 2;
                      break;
                  case BT_LEAD3:
                      if (end - off < 3)
                          throw new PartialCharException(off);
                      if (byteType3(buf, off) != BT_NMSTRT)
                          throw new InvalidTokenException(off);
                      off += 3;
                      break;
                  case BT_LEAD4:
                      if (end - off < 4)
                          throw new PartialCharException(off);
                      if (byteType4(buf, off) != BT_NMSTRT)
                          throw new InvalidTokenException(off);
                      off += 4;
                      break;
                  default:
                      throw new InvalidTokenException(off);
              }
              while (off != end)
              {
                  switch (byteType(buf, off))
                  {
                      case BT_NMSTRT:
                      case BT_NAME:
                      case BT_MINUS:
                          off += minBPC;
                          break;
                      case BT_LEAD2:
                          if (end - off < 2)
                              throw new PartialCharException(off);
                          if (!isNameChar2(buf, off))
                              throw new InvalidTokenException(off);
                          off += 2;
                          break;
                      case BT_LEAD3:
                          if (end - off < 3)
                              throw new PartialCharException(off);
                          if (!isNameChar3(buf, off))
                              throw new InvalidTokenException(off);
                          off += 3;
                          break;
                      case BT_LEAD4:
                          if (end - off < 4)
                              throw new PartialCharException(off);
                          if (!isNameChar4(buf, off))
                              throw new InvalidTokenException(off);
                          off += 4;
                          break;
                      case BT_S:
                      case BT_CR:
                      case BT_LF:
                          bool isXml = targetIsXml(buf, target, off);
                          token.NameEnd = off;
                          off += minBPC;
                          while (off != end)
                          {
                              switch (byteType(buf, off))
                              {
                                  case BT_LEAD2:
                                      if (end - off < 2)
                                          throw new PartialCharException(off);
                                      check2(buf, off);
                                      off += 2;
                                      break;
                                  case BT_LEAD3:
                                      if (end - off < 3)
                                          throw new PartialCharException(off);
                                      check3(buf, off);
                                      off += 3;
                                      break;
                                  case BT_LEAD4:
                                      if (end - off < 4)
                                          throw new PartialCharException(off);
                                      check4(buf, off);
                                      off += 4;
                                      break;
                                  case BT_NONXML:
                                  case BT_MALFORM:
                                      throw new InvalidTokenException(off);
                                  case BT_QUEST:
                                      off += minBPC;
                                      if (off == end)
                                          throw new PartialTokenException();
                                      if (charMatches(buf, off, '>'))
                                      {
                                          token.TokenEnd = off + minBPC;
                                          if (isXml)
                                              return TOK.XML_DECL;
                                          else
                                              return TOK.PI;
                                      }
                                      break;
                                  default:
                                      off += minBPC;
                                      break;
                              }
                          }
                          throw new PartialTokenException();
                      case BT_QUEST:
                          token.NameEnd = off;
                          off += minBPC;
                          if (off == end)
                              throw new PartialTokenException();
                          checkCharMatches(buf, off, '>');
                          token.TokenEnd = off + minBPC;
                          return (targetIsXml(buf, target, token.NameEnd)
                              ? TOK.XML_DECL
                              : TOK.PI);
                      default:
                          throw new InvalidTokenException(off);
                  }
              }
              throw new PartialTokenException();
          }
  
          /* off points to character following "<![" */
          private const string CDATA = "CDATA[";
  
          private TOK scanCdataSection(byte[] buf, int off, int end, Token token)
          {
              /* "CDATA[".length() == 6 */
              if (end - off < 6*minBPC)
                  throw new PartialTokenException();
              for (int i = 0; i < CDATA.Length; i++, off += minBPC)
                  checkCharMatches(buf, off, CDATA[i]);
              token.TokenEnd = off;
              return TOK.CDATA_SECT_OPEN;
          }
  
          /**
           * Scans the first token of a byte subarrary that starts with the
           * content of a CDATA section.
           * Returns one of the following integers according to the type of token
           * that the subarray starts with:
           * <ul>
           * <li><code>TOK.DATA_CHARS</code></li>
           * <li><code>TOK.DATA_NEWLINE</code></li>
           * <li><code>TOK.CDATA_SECT_CLOSE</code></li>
           * </ul>
           * <p>
           * Information about the token is stored in <code>token</code>.
           * </p>
           * After <code>TOK.CDATA_SECT_CLOSE</code> is returned, the application
           * should use <code>tokenizeContent</code>.
           *
           * @exception EmptyTokenException if the subarray is empty
           * @exception PartialTokenException if the subarray contains only part of
           * a legal token
           * @exception InvalidTokenException if the subarrary does not start
           * with a legal token or part of one
           * @exception ExtensibleTokenException if the subarray encodes just a carriage
           * return ('\r')
           *
           * @see #TOK.DATA_CHARS
           * @see #TOK.DATA_NEWLINE
           * @see #TOK.CDATA_SECT_CLOSE
           * @see Token
           * @see EmptyTokenException
           * @see PartialTokenException
           * @see InvalidTokenException
           * @see ExtensibleTokenException
           * @see #tokenizeContent
           */
  
          public TOK tokenizeCdataSection(byte[] buf, int off, int end,
              Token token)
          {
              if (minBPC > 1)
                  end = adjustEnd(off, end);
              if (off == end)
                  throw new EmptyTokenException();
              switch (byteType(buf, off))
              {
                  case BT_RSQB:
                      off += minBPC;
                      if (off == end)
  
gt:                         throw new PartialTokenException();