TweetTokeniser implements Iterable<Token> { public List<Token> getTokens() { return this.tokenize; } TweetTokeniser(String s); static boolean isValid(Locale locale); static boolean isValid(String locale); @Override Iterator<Token> iterator(); List<Token> getTokens(); List<String> getStringTokens(); List<String> getProtectedStringTokens(); List<String> getUnprotectedStringTokens();  }
@Test public void newlines() throws UnsupportedEncodingException, TweetTokeniserException { final String[] tweets = new String[] { "words\r\nacross new\nlines", }; for (final String text : tweets) { final TweetTokeniser tokeniser = new TweetTokeniser(text); System.out.println("Tweet: " + text); final String tokens = "[" + StringUtils.join(tokeniser.getTokens(), ",") + "]"; System.out.println("Tokens: " + tokens); } }
@Test public void testSingleTweets() throws UnsupportedEncodingException, TweetTokeniserException { final String[] tweets = new String[] { "@janecds RT _badbristal np VYBZ KARTEL - TURN & WINE&lt; WE DANCEN TO THIS LOL? http: for (final String text : tweets) { final TweetTokeniser tokeniser = new TweetTokeniser(text); System.out.println("Tweet: " + text); final String tokens = "[" + StringUtils.join(tokeniser.getTokens(), ",") + "]"; System.out.println("Tokens: " + tokens); } }