TweetTokeniser implements Iterable<Token> { public List<Token> getTokens(){ return this.tokenize; } TweetTokeniser(String s); static boolean isValid(Locale locale); static boolean isValid(String locale); @Override Iterator<Token> iterator(); List<Token> getTokens(); List<String> getStringTokens(); List<String> getProtectedStringTokens(); List<String> getUnprotectedStringTokens();  }
@Test public void newlines() throws UnsupportedEncodingException, TweetTokeniserException{ EmoticonPatternProvider provider = new EmoticonPatternProvider(); String[] tweets = new String[]{ "words\r\nacross new\nlines", }; for (String text: tweets) { TweetTokeniser tokeniser = new TweetTokeniser(text); System.out.println("Tweet: " + text); String tokens = "[" + StringUtils.join(tokeniser.getTokens(), ",") + "]"; System.out.println("Tokens: " + tokens); } }
@Test public void testSingleTweets() throws UnsupportedEncodingException, TweetTokeniserException{ String[] tweets = new String[]{ "@janecds RT _badbristal np VYBZ KARTEL - TURN & WINE&lt; WE DANCEN TO THIS LOL? http: }; for (String text: tweets) { TweetTokeniser tokeniser = new TweetTokeniser(text); System.out.println("Tweet: " + text); String tokens = "[" + StringUtils.join(tokeniser.getTokens(), ",") + "]"; System.out.println("Tokens: " + tokens); } }