Tokenizer implements Serializable { public List<String> tokenizeString(String inputString){ List<String> tokenList = new ArrayList<String>(); StringTokenizer st = new StringTokenizer(inputString); while (st.hasMoreTokens()) { String currentToken = st.nextToken(); currentToken = currentToken.toLowerCase(); for (String i : separator) { if (currentToken.contains(i)) { currentToken = currentToken.replace(i, ""); } } for (String p : doubleList) { if (currentToken.contains(p)) { String additionalHashtagToken = currentToken.replace(p, ""); if(additionalHashtagToken.length() >= 2) { tokenList.add(additionalHashtagToken); } } } if(currentToken.length()>=2) { tokenList.add(currentToken); } } return tokenList; } Tokenizer(); List<String> tokenizeString(String inputString);  }
@Test public void testGetTokens(){ Tokenizer tokenizer = new Tokenizer(); List<String> tokens1 = tokenizer.tokenizeString(INPUT_TEXT1); for(String s: tokens1){ System.out.println(s); Boolean equal1 = false; for(int i= 0 ; i<TOKENS1.length;i++){ if(s.equals(TOKENS1[i])){ equal1= true; break; } } Assert.assertTrue(equal1); } List<String> tokens2 = tokenizer.tokenizeString(INPUT_TEXT2); for(String s: tokens2){ Boolean equal2 = false; for(int i= 0 ; i<TOKENS2.length;i++){ if(s.equals(TOKENS2[i])){ equal2= true; break; } } Assert.assertTrue(equal2); } }