@Test void testTokenizeSentenceWithContraction() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("I'm an engineer"); assertEquals(3, results.size()); assertEquals("I'm", results.get(0).getSurface()); assertEquals("an", results.get(1).getSurface()); assertEquals("engineer", results.get(2).getSurface()); }
@Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); @Override List<TokenElement> tokenize(String content); }
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); @Override List<TokenElement> tokenize(String content);  }