@Test public void shouldTokenizeTextWithExceptionalCase_5() { { final String text = "5:20 gibi gelecek."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5:20", "gibi", "gelecek", "."))); } { final String text = "5,60 TL verdim."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5,60", "TL", "verdim", "."))); } { final String text = "5.600 TL verdim."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5.600", "TL", "verdim", "."))); } { final String text = "5.600,1234 TL verdim."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5.600,1234", "TL", "verdim", "."))); } { final String text = "100'le gidiyor."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("100\'le", "gidiyor", "."))); } { final String text = "5:20'de geliyorum."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5:20\'de", "geliyorum", "."))); } { final String text = "5,74'te bir ihtimal."; final Iterable<String> tokens = getSurfaces(tokenizer.tokenize(text)); assertThat(Lists.newArrayList(tokens), equalTo(Arrays.asList("5,74\'te", "bir", "ihtimal", "."))); } }
public LinkedList<Token> tokenize(String text) { if (logger.isDebugEnabled()) logger.debug("Tokenizing text: '" + text + "'"); text = text.replaceAll(" +", " "); text = text.trim(); final LinkedList<TextBlock> textBlocks = textBlockSplitter.splitToTextParts(text); this.textBlockSplitter.addTextStartsAndEnds(textBlocks, this.blockSize); final LinkedList<Token> tokens = new LinkedList<Token>(); StringBuilder currentTokenBuilder = new StringBuilder(); List<TextBlockType> currentBlockTypes = new LinkedList<TextBlockType>(); for (int i = this.blockSize; i <= textBlocks.size() - this.blockSize; i++) { final TextBlockGroup leftTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i - this.blockSize); final TextBlockGroup rightTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i); if (logger.isDebugEnabled()) logger.debug("Applying rule for left : " + leftTextBlockGroup.getTextBlockTypeGroup() + " right :" + rightTextBlockGroup.getTextBlockTypeGroup()); boolean addSpace; try { addSpace = this.graph.isAddSpace(leftTextBlockGroup, rightTextBlockGroup, textBlocks, i); if (this.stats != null) this.stats.addSuccess(leftTextBlockGroup, rightTextBlockGroup); } catch (MissingTokenizationRuleException ex) { if (strict) { throw ex; } else { addSpace = false; if (this.stats != null) this.stats.addFail(ex); } } final TextBlock firstTextBlock = rightTextBlockGroup.getFirstTextBlock(); final String textToAdd = firstTextBlock.getText(); final TextBlockType textBlockType = firstTextBlock.getTextBlockType(); if (addSpace || SPACE.equals(textToAdd)) { if (currentTokenBuilder.length() > 0){ tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); } if (SPACE.equals(textToAdd)){ currentTokenBuilder = new StringBuilder(); currentBlockTypes = new LinkedList<TextBlockType>(); } else{ currentTokenBuilder = new StringBuilder(textToAdd); currentBlockTypes.add(textBlockType); } } else { currentTokenBuilder.append(textToAdd); currentBlockTypes.add(textBlockType); } } if (currentTokenBuilder.length() > 0) tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); return tokens; }
TextTokenizer { public LinkedList<Token> tokenize(String text) { if (logger.isDebugEnabled()) logger.debug("Tokenizing text: '" + text + "'"); text = text.replaceAll(" +", " "); text = text.trim(); final LinkedList<TextBlock> textBlocks = textBlockSplitter.splitToTextParts(text); this.textBlockSplitter.addTextStartsAndEnds(textBlocks, this.blockSize); final LinkedList<Token> tokens = new LinkedList<Token>(); StringBuilder currentTokenBuilder = new StringBuilder(); List<TextBlockType> currentBlockTypes = new LinkedList<TextBlockType>(); for (int i = this.blockSize; i <= textBlocks.size() - this.blockSize; i++) { final TextBlockGroup leftTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i - this.blockSize); final TextBlockGroup rightTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i); if (logger.isDebugEnabled()) logger.debug("Applying rule for left : " + leftTextBlockGroup.getTextBlockTypeGroup() + " right :" + rightTextBlockGroup.getTextBlockTypeGroup()); boolean addSpace; try { addSpace = this.graph.isAddSpace(leftTextBlockGroup, rightTextBlockGroup, textBlocks, i); if (this.stats != null) this.stats.addSuccess(leftTextBlockGroup, rightTextBlockGroup); } catch (MissingTokenizationRuleException ex) { if (strict) { throw ex; } else { addSpace = false; if (this.stats != null) this.stats.addFail(ex); } } final TextBlock firstTextBlock = rightTextBlockGroup.getFirstTextBlock(); final String textToAdd = firstTextBlock.getText(); final TextBlockType textBlockType = firstTextBlock.getTextBlockType(); if (addSpace || SPACE.equals(textToAdd)) { if (currentTokenBuilder.length() > 0){ tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); } if (SPACE.equals(textToAdd)){ currentTokenBuilder = new StringBuilder(); currentBlockTypes = new LinkedList<TextBlockType>(); } else{ currentTokenBuilder = new StringBuilder(textToAdd); currentBlockTypes.add(textBlockType); } } else { currentTokenBuilder.append(textToAdd); currentBlockTypes.add(textBlockType); } } if (currentTokenBuilder.length() > 0) tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); return tokens; } }
TextTokenizer { public LinkedList<Token> tokenize(String text) { if (logger.isDebugEnabled()) logger.debug("Tokenizing text: '" + text + "'"); text = text.replaceAll(" +", " "); text = text.trim(); final LinkedList<TextBlock> textBlocks = textBlockSplitter.splitToTextParts(text); this.textBlockSplitter.addTextStartsAndEnds(textBlocks, this.blockSize); final LinkedList<Token> tokens = new LinkedList<Token>(); StringBuilder currentTokenBuilder = new StringBuilder(); List<TextBlockType> currentBlockTypes = new LinkedList<TextBlockType>(); for (int i = this.blockSize; i <= textBlocks.size() - this.blockSize; i++) { final TextBlockGroup leftTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i - this.blockSize); final TextBlockGroup rightTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i); if (logger.isDebugEnabled()) logger.debug("Applying rule for left : " + leftTextBlockGroup.getTextBlockTypeGroup() + " right :" + rightTextBlockGroup.getTextBlockTypeGroup()); boolean addSpace; try { addSpace = this.graph.isAddSpace(leftTextBlockGroup, rightTextBlockGroup, textBlocks, i); if (this.stats != null) this.stats.addSuccess(leftTextBlockGroup, rightTextBlockGroup); } catch (MissingTokenizationRuleException ex) { if (strict) { throw ex; } else { addSpace = false; if (this.stats != null) this.stats.addFail(ex); } } final TextBlock firstTextBlock = rightTextBlockGroup.getFirstTextBlock(); final String textToAdd = firstTextBlock.getText(); final TextBlockType textBlockType = firstTextBlock.getTextBlockType(); if (addSpace || SPACE.equals(textToAdd)) { if (currentTokenBuilder.length() > 0){ tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); } if (SPACE.equals(textToAdd)){ currentTokenBuilder = new StringBuilder(); currentBlockTypes = new LinkedList<TextBlockType>(); } else{ currentTokenBuilder = new StringBuilder(textToAdd); currentBlockTypes.add(textBlockType); } } else { currentTokenBuilder.append(textToAdd); currentBlockTypes.add(textBlockType); } } if (currentTokenBuilder.length() > 0) tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); return tokens; } private  TextTokenizer(TextTokenizerBuilder builder); }
TextTokenizer { public LinkedList<Token> tokenize(String text) { if (logger.isDebugEnabled()) logger.debug("Tokenizing text: '" + text + "'"); text = text.replaceAll(" +", " "); text = text.trim(); final LinkedList<TextBlock> textBlocks = textBlockSplitter.splitToTextParts(text); this.textBlockSplitter.addTextStartsAndEnds(textBlocks, this.blockSize); final LinkedList<Token> tokens = new LinkedList<Token>(); StringBuilder currentTokenBuilder = new StringBuilder(); List<TextBlockType> currentBlockTypes = new LinkedList<TextBlockType>(); for (int i = this.blockSize; i <= textBlocks.size() - this.blockSize; i++) { final TextBlockGroup leftTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i - this.blockSize); final TextBlockGroup rightTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i); if (logger.isDebugEnabled()) logger.debug("Applying rule for left : " + leftTextBlockGroup.getTextBlockTypeGroup() + " right :" + rightTextBlockGroup.getTextBlockTypeGroup()); boolean addSpace; try { addSpace = this.graph.isAddSpace(leftTextBlockGroup, rightTextBlockGroup, textBlocks, i); if (this.stats != null) this.stats.addSuccess(leftTextBlockGroup, rightTextBlockGroup); } catch (MissingTokenizationRuleException ex) { if (strict) { throw ex; } else { addSpace = false; if (this.stats != null) this.stats.addFail(ex); } } final TextBlock firstTextBlock = rightTextBlockGroup.getFirstTextBlock(); final String textToAdd = firstTextBlock.getText(); final TextBlockType textBlockType = firstTextBlock.getTextBlockType(); if (addSpace || SPACE.equals(textToAdd)) { if (currentTokenBuilder.length() > 0){ tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); } if (SPACE.equals(textToAdd)){ currentTokenBuilder = new StringBuilder(); currentBlockTypes = new LinkedList<TextBlockType>(); } else{ currentTokenBuilder = new StringBuilder(textToAdd); currentBlockTypes.add(textBlockType); } } else { currentTokenBuilder.append(textToAdd); currentBlockTypes.add(textBlockType); } } if (currentTokenBuilder.length() > 0) tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); return tokens; } private  TextTokenizer(TextTokenizerBuilder builder); LinkedList<Token> tokenize(String text); TextTokenizerStats getStats(); @SuppressWarnings("UnusedDeclaration") static TextTokenizer createDefaultTextTokenizer(); static TextTokenizer createDefaultTextTokenizer(boolean recordExamples); static TextTokenizerBuilder newBuilder(); TokenizationGraph getGraph(); }
TextTokenizer { public LinkedList<Token> tokenize(String text) { if (logger.isDebugEnabled()) logger.debug("Tokenizing text: '" + text + "'"); text = text.replaceAll(" +", " "); text = text.trim(); final LinkedList<TextBlock> textBlocks = textBlockSplitter.splitToTextParts(text); this.textBlockSplitter.addTextStartsAndEnds(textBlocks, this.blockSize); final LinkedList<Token> tokens = new LinkedList<Token>(); StringBuilder currentTokenBuilder = new StringBuilder(); List<TextBlockType> currentBlockTypes = new LinkedList<TextBlockType>(); for (int i = this.blockSize; i <= textBlocks.size() - this.blockSize; i++) { final TextBlockGroup leftTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i - this.blockSize); final TextBlockGroup rightTextBlockGroup = this.textBlockSplitter.getTextBlockGroup(textBlocks, this.blockSize, i); if (logger.isDebugEnabled()) logger.debug("Applying rule for left : " + leftTextBlockGroup.getTextBlockTypeGroup() + " right :" + rightTextBlockGroup.getTextBlockTypeGroup()); boolean addSpace; try { addSpace = this.graph.isAddSpace(leftTextBlockGroup, rightTextBlockGroup, textBlocks, i); if (this.stats != null) this.stats.addSuccess(leftTextBlockGroup, rightTextBlockGroup); } catch (MissingTokenizationRuleException ex) { if (strict) { throw ex; } else { addSpace = false; if (this.stats != null) this.stats.addFail(ex); } } final TextBlock firstTextBlock = rightTextBlockGroup.getFirstTextBlock(); final String textToAdd = firstTextBlock.getText(); final TextBlockType textBlockType = firstTextBlock.getTextBlockType(); if (addSpace || SPACE.equals(textToAdd)) { if (currentTokenBuilder.length() > 0){ tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); } if (SPACE.equals(textToAdd)){ currentTokenBuilder = new StringBuilder(); currentBlockTypes = new LinkedList<TextBlockType>(); } else{ currentTokenBuilder = new StringBuilder(textToAdd); currentBlockTypes.add(textBlockType); } } else { currentTokenBuilder.append(textToAdd); currentBlockTypes.add(textBlockType); } } if (currentTokenBuilder.length() > 0) tokens.add(new Token(currentTokenBuilder.toString(), currentBlockTypes)); return tokens; } private  TextTokenizer(TextTokenizerBuilder builder); LinkedList<Token> tokenize(String text); TextTokenizerStats getStats(); @SuppressWarnings("UnusedDeclaration") static TextTokenizer createDefaultTextTokenizer(); static TextTokenizer createDefaultTextTokenizer(boolean recordExamples); static TextTokenizerBuilder newBuilder(); TokenizationGraph getGraph();  }