code: /* LanguageTool, a natural language style checker 
   * Copyright (C) 2005 Daniel Naber (http://www.danielnaber.de)
   * 
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
   * version 2.1 of the License, or (at your option) any later version.
   *
   * This library is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with this library; if not, write to the Free Software
   * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
   * USA
   */
  package org.languagetool.dev.index;
  
  import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
  import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
  import org.apache.lucene.analysis.Tokenizer;
  import org.apache.lucene.analysis.util.CharacterUtils;
  import org.apache.lucene.util.AttributeFactory;
  
  import java.io.IOException;
  
  /**
   * A tokenizer that renders the whole input as one token.
   * 
   * @author Tao Lin
   */
  public final class AnyCharTokenizer extends Tokenizer {
  
    private static final int MAX_WORD_LEN = Integer.MAX_VALUE; // extend the word length!
  
    private final CharacterUtils.CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(4096);
    private final CharacterUtils charUtils = CharacterUtils.getInstance();
    private final CharTermAttribute termAtt = this.addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = this.addAttribute(OffsetAttribute.class);
  
    private int bufferIndex = 0;
    private int dataLen = 0;
    private int offset = 0;
    private int finalOffset = 0;
    
    /**
     * Construct a new AnyCharTokenizer.
     */
    public AnyCharTokenizer() {
      super();
    }
  
    /**
     * Construct a new AnyCharTokenizer using a given
     * {@link org.apache.lucene.util.AttributeFactory}.
     * @param factory the attribute factory to use for this {@link org.apache.lucene.analysis.Tokenizer}
     */
    public AnyCharTokenizer(AttributeFactory factory) {
      super(factory);
    }
  
    /**
     * Collects any characters.
     */
    protected boolean isTokenChar(int c) {
      return true;
    }
  
    protected int normalize(int c) {
      return c;
    }
  
    @Override
    public boolean incrementToken() throws IOException {
      this.clearAttributes();
      int length = 0;
      int start = -1;
      int end = -1;
      char[] buffer = this.termAtt.buffer();
  
      while(true) {
        if(this.bufferIndex >= this.dataLen) {
          this.offset += this.dataLen;
          this.charUtils.fill(this.ioBuffer, this.input);
          if(this.ioBuffer.getLength() == 0) {
            this.dataLen = 0;
            if(length <= 0) {
              this.finalOffset = this.correctOffset(this.offset);
              return false;
            }
            break;
          }
  
          this.dataLen = this.ioBuffer.getLength();
          this.bufferIndex = 0;
        }
  
        int c = this.charUtils.codePointAt(this.ioBuffer.getBuffer(), this.bufferIndex, this.ioBuffer.getLength());
        int charCount = Character.charCount(c);
        this.bufferIndex += charCount;
        if(this.isTokenChar(c)) {
          if(length == 0) {
            assert start == -1;
  
            start = this.offset + this.bufferIndex - charCount;
            end = start;
          } else if(length >= buffer.length - 1) {
            buffer = this.termAtt.resizeBuffer(2 + length);
          }
  
          end += charCount;
          length += Character.toChars(this.normalize(c), buffer, length);
          if(length >= MAX_WORD_LEN) {
            break;
          }
        } else if(length > 0) {
          break;
        }
      }
  
      this.termAtt.setLength(length);
  
      assert start != -1;
  
      this.offsetAtt.setOffset(this.correctOffset(start), this.finalOffset = this.correctOffset(end));
      return true;
    }
  
    @Override
    public void end() throws IOException {
      super.end();
      this.offsetAtt.setOffset(this.finalOffset, this.finalOffset);
    }
  
    @Override
    public void reset() throws IOException {
      super.reset();
      this.bufferIndex = 0;
      this.offset = 0;
      this.dataLen = 0;
      this.finalOffset = 0;
      this.ioBuffer.reset();
    }
  
  }
  