WordTokenizer implements Tokenizer { @Override public int findTokenStart(final @NonNull Spanned text, final int cursor) { int start = getSearchStartIndex(text, cursor); int i = cursor; if (isExplicit(text, cursor)) { i--; while (i >= start) { char currentChar = text.charAt(i); if (isExplicitChar(currentChar)) { if (i == 0 || isWordBreakingChar(text.charAt(i - 1))) { return i; } } i--; } return -1; } else { int maxNumKeywords = mConfig.MAX_NUM_KEYWORDS; while (i > start && !isWordBreakingChar(text.charAt(i - 1))) { i--; } for (int j = 0; j < maxNumKeywords - 1; j++) { if (i > start && isWordBreakingChar(text.charAt(i - 1))) { i--; } if (i > start && isWordBreakingChar(text.charAt(i - 1))) { break; } while (i > start && !isWordBreakingChar(text.charAt(i - 1))) { i--; } } while (i < cursor && (isWordBreakingChar(text.charAt(i)) || isExplicitChar(text.charAt(i)))) { i++; } return i; } } WordTokenizer();  WordTokenizer(final @NonNull WordTokenizerConfig config); @Override int findTokenStart(final @NonNull Spanned text, final int cursor); @Override int findTokenEnd(final @NonNull Spanned text, final int cursor); boolean isValidMention(final @NonNull Spanned text, final int start, final int end); @Override @NonNull Spanned terminateToken(final @NonNull Spanned text); boolean isExplicitChar(final char c); boolean isWordBreakingChar(final char c); boolean isExplicit(final @NonNull CharSequence text, final int cursor); char getExplicitChar(final @NonNull CharSequence text, final int cursor); boolean containsExplicitChar(final @NonNull CharSequence input); boolean containsWordBreakingChar(final @NonNull CharSequence input); boolean onlyLettersOrDigits(final @NonNull CharSequence input, final int numCharsToCheck, final int start);  }
@Test public void testFindTokenStart() { findTokenStartChecker(0, 0); findTokenStartChecker(1, 0); findTokenStartChecker(2, 0); findTokenStartChecker(3, 0); findTokenStartChecker(4, 0); findTokenStartChecker(5, 0); findTokenStartChecker(6, 0); findTokenStartChecker(13, 0); findTokenStartChecker(14, 5); findTokenStartChecker(15, 5); findTokenStartChecker(22, 5); findTokenStartChecker(23, 14); findTokenStartChecker(24, 14); findTokenStartChecker(25, 14); findTokenStartChecker(26, 23); findTokenStartChecker(27, 23); setTestSpan("Shoulong Li", 14); findTokenStartChecker(25, 25); findTokenStartChecker(26, 26); findTokenStartChecker(27, 26); findTokenStartChecker(30, 26); findTokenStartChecker(31, 30); findTokenStartChecker(38, 30); findTokenStartChecker(41, 38); findTokenStartChecker(42, 41); findTokenStartChecker(43, 41); }