TweetParser { protected Map<String, Integer> tokenize(String text) { Map<String, Integer> tokenMap = newHashMap(); StringTokenizer st = new StringTokenizer(text, "\"{}[]:;|<>?`'.,/~!@#$%^&*()_-+= \t\n\r\f\\"); while (st.hasMoreTokens()) { String token = st.nextToken(); if (token.length() < MIN_TOKEN_LENGTH) { continue; } Integer count = tokenMap.containsKey(token) ? tokenMap.get(token) + 1 : 1; tokenMap.put(token, count); } return tokenMap; }  @SpaceDataEvent SpaceDocument eventListener(SpaceDocument tweet);  }
@Test public void tokenize() { String text = "Twitter is an online social networking service and microblogging service" + " that enables its users to send and read text-based posts of up to 140 characters," + " known as \"tweets\". You can signup and test it yourself."; Map<String, Integer> tokenMap = new TweetParser().tokenize(text); assertThat(tokenMap).hasSize(25); assertThat(tokenMap.get("microblogging")).isEqualTo(1); assertThat(tokenMap.get("and")).isEqualTo(3); assertThat(tokenMap.get("unknown")).isNull(); for (Entry<String, Integer> e : tokenMap.entrySet()) { System.out.println(e.getKey() + ": " + e.getValue()); } }