@Test void testTokenizeSentenceWithNoSpaceBracket() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("distributed(cluster) systems are good"); assertEquals(7, results.size()); assertEquals("distributed", results.get(0).getSurface()); assertEquals(0, results.get(0).getTags().size()); assertEquals("cluster", results.get(2).getSurface()); assertEquals(0, results.get(1).getTags().size()); assertEquals("systems", results.get(4).getSurface()); assertEquals(0, results.get(2).getTags().size()); assertEquals("are", results.get(5).getSurface()); assertEquals(0, results.get(3).getTags().size()); assertEquals("good", results.get(6).getSurface()); assertEquals(0, results.get(4).getTags().size()); }
@Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); @Override List<TokenElement> tokenize(String content); }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); @Override List<TokenElement> tokenize(String content);  }