code: /* LanguageTool, a natural language style checker 
   * Copyright (C) 2016 Daniel Naber (http://www.danielnaber.de)
   * 
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
   * version 2.1 of the License, or (at your option) any later version.
   *
   * This library is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with this library; if not, write to the Free Software
   * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
   * USA
   */
  package org.languagetool.rules.ngrams;
  
  import org.junit.Test;
  import org.languagetool.*;
  import org.languagetool.language.Demo;
  import org.languagetool.tagging.Tagger;
  import org.languagetool.tokenizers.WordTokenizer;
  
  import java.io.IOException;
  import java.util.*;
  
  import static org.hamcrest.CoreMatchers.is;
  import static org.hamcrest.MatcherAssert.assertThat;
  
  public class GoogleTokenTest {
    
    @Test
    public void testTokenization() {
      List<GoogleToken> tokens = GoogleToken.getGoogleTokens("This, isn't a test.", false, new MyWordTokenizer());
      assertThat(tokens.get(0).token, is("This"));
      assertThat(tokens.get(0).posTags.toString(), is("[]"));
      assertThat(tokens.get(1).token, is(","));
      assertThat(tokens.get(2).token, is("isn"));
      assertThat(tokens.get(3).token, is("'t"));
      assertThat(tokens.get(4).token, is("a"));
      assertThat(tokens.get(5).token, is("test"));
      assertThat(tokens.get(6).token, is("."));
    }
  
    @Test
    public void testTokenizationWithPosTag() throws IOException {
      JLanguageTool lt = new JLanguageTool(new PosTaggingDemo());
      AnalyzedSentence analyzedSentence = lt.getAnalyzedSentence("This, isn't a test.");
      List<GoogleToken> tokens = GoogleToken.getGoogleTokens(analyzedSentence, false, new MyWordTokenizer());
      assertThat(tokens.get(0).token, is("This"));
      assertThat(tokens.get(0).posTags.toString(), is("[This/DT]"));
      assertThat(tokens.get(1).token, is(","));
      assertThat(tokens.get(1).posTags.toString(), is("[,/null]"));
      assertThat(tokens.get(2).token, is("isn"));
      assertThat(tokens.get(3).token, is("'t"));
      assertThat(tokens.get(4).token, is("a"));
      assertThat(tokens.get(5).token, is("test"));
      assertThat(tokens.get(5).posTags.toString(), is("[test/NN]"));
      assertThat(tokens.get(6).token, is("."));
    }
  
    class PosTaggingDemo extends Demo {
      @Override
      public Tagger createDefaultTagger() {
        return new Tagger() {
          @Override
          public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens) {
            List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
            int pos = 0;
            for (String word : sentenceTokens) {
              List<AnalyzedToken> l = new ArrayList<>();
              switch (word) {
                case "This": l.add(new AnalyzedToken(word, "DT", word)); break;
                case "is": l.add(new AnalyzedToken(word, "VBZ", word)); break;
                case "test": l.add(new AnalyzedToken(word, "NN", word)); break;
                default: l.add(new AnalyzedToken(word, null, word)); break;
              }
              tokenReadings.add(new AnalyzedTokenReadings(l, pos));
              pos += word.length();
            }
            return tokenReadings;
          }
          @Override
          public AnalyzedTokenReadings createNullToken(String token, int startPos) {
            return null;
          }
          @Override
          public AnalyzedToken createToken(String token, String posTag) {
            return null;
          }
        };
      }
    }
  
    private class MyWordTokenizer extends WordTokenizer {
      @Override
      public List<String> tokenize(String text) {
        List<String> tokens = super.tokenize(text);
        String prev = null;
        Stack<String> l = new Stack<>();
        for (String token : tokens) {
          if ("'".equals(prev)) {
            if (token.equals("t")) {
              l.pop();
              l.push("'t");
            } else {
              l.push(token);
            }
          } else {
            l.push(token);
          }
          prev = token;
        }
        return l;
      }
    }
  
  }
  