TagTokenizer extends Tokenizer { public void addField(String f) { whitelist.add(Pattern.compile(f)); } TagTokenizer();  TagTokenizer(Parameters tokenizerParameters);  TagTokenizer(TupleFlowParameters parameters); void addField(String f); void reset(); @Override void tokenize(Document document); ArrayList<IntSpan> getTokenPositions(); static final Logger log; static HashSet<String> ignoredTags; public TagTokenizerParser state; }
@Test public void testDoNotTokenizeAttributes() throws IncompatibleProcessorException, IOException { TagTokenizer tokenizer = new TagTokenizer(); tokenizer.addField("title"); tokenizer.addField("category"); tokenizer.addField("anchor"); Document document = new Document(); tokenizer.setProcessor(new NullProcessor(Document.class)); document.text = "<document> " + "<title>Aberdeen</title> \n" + "<category tokenizeTagContent=\"false\">/location/citytown</category>\n" + "<category tokenizeTagContent=\"false\">/base/scotland/topic</category>\n" + "<anchor tokenizeTagContent=\"false\">Aberdeen Angus</anchor>\n" + "</html>"; tokenizer.process(document); String[] tokens = {"aberdeen", "/location/citytown", "/base/scotland/topic", "aberdeen angus"}; assertEquals("Token length", tokens.length, document.terms.size()); for (int i = 0; i < tokens.length; i++) { assertEquals("Token text", tokens[i], document.terms.get(i)); } assertEquals(4, document.tags.size()); }