@Test public void testTokenStreamToStringWithDelimiter() throws Exception{ TokenStreamStringConcatenator tokenStreamStringConcatenator = new TokenStreamStringConcatenator(); Collection<Token> tokens1 = new ArrayList<Token>(); tokens1.add(new Token("token1".toCharArray(),0,6,0,6)); tokens1.add(new Token("token2".toCharArray(),0,6,7,13)); tokens1.add(new Token("token3".toCharArray(),0,6,14,20)); CollectionTokenStream tokenStream1 = new CollectionTokenStream(tokens1); String concatenatedString = tokenStreamStringConcatenator.tokenStreamToStringWithDelimiter(tokenStream1, " "); assertEquals("token1 token2 token3", concatenatedString); }
public String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter) throws IOException { String tokenString = ""; Token newToken = new Token(); while (true) { Token token = ts.next(newToken); if (token != null) { tokenString = tokenString.concat(new String(token.termBuffer(), 0, token.termLength())) .concat(delimiter); } else { int lastIndex = tokenString.lastIndexOf(delimiter); if (lastIndex >= 0) { tokenString = tokenString.substring(0, lastIndex); } break; } } return tokenString; }
TokenStreamStringConcatenator { public String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter) throws IOException { String tokenString = ""; Token newToken = new Token(); while (true) { Token token = ts.next(newToken); if (token != null) { tokenString = tokenString.concat(new String(token.termBuffer(), 0, token.termLength())) .concat(delimiter); } else { int lastIndex = tokenString.lastIndexOf(delimiter); if (lastIndex >= 0) { tokenString = tokenString.substring(0, lastIndex); } break; } } return tokenString; } }
TokenStreamStringConcatenator { public String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter) throws IOException { String tokenString = ""; Token newToken = new Token(); while (true) { Token token = ts.next(newToken); if (token != null) { tokenString = tokenString.concat(new String(token.termBuffer(), 0, token.termLength())) .concat(delimiter); } else { int lastIndex = tokenString.lastIndexOf(delimiter); if (lastIndex >= 0) { tokenString = tokenString.substring(0, lastIndex); } break; } } return tokenString; }  }
TokenStreamStringConcatenator { public String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter) throws IOException { String tokenString = ""; Token newToken = new Token(); while (true) { Token token = ts.next(newToken); if (token != null) { tokenString = tokenString.concat(new String(token.termBuffer(), 0, token.termLength())) .concat(delimiter); } else { int lastIndex = tokenString.lastIndexOf(delimiter); if (lastIndex >= 0) { tokenString = tokenString.substring(0, lastIndex); } break; } } return tokenString; }  String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter); }
TokenStreamStringConcatenator { public String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter) throws IOException { String tokenString = ""; Token newToken = new Token(); while (true) { Token token = ts.next(newToken); if (token != null) { tokenString = tokenString.concat(new String(token.termBuffer(), 0, token.termLength())) .concat(delimiter); } else { int lastIndex = tokenString.lastIndexOf(delimiter); if (lastIndex >= 0) { tokenString = tokenString.substring(0, lastIndex); } break; } } return tokenString; }  String tokenStreamToStringWithDelimiter(TokenStream ts, String delimiter);  }