TagTokenizer extends Tokenizer { @Override public void tokenize(Document document) { reset(); assert(document != null); state.text = document.text; assert(state.text != null); try { state.parse(); state.finishDocument(document, whitelist); } catch (Exception e) { log.log(Level.WARNING, "Parse failure: " + document.name, e); } assert(document.terms != null); StringPooler.getInstance().transform(document.terms); } TagTokenizer();  TagTokenizer(Parameters tokenizerParameters);  TagTokenizer(TupleFlowParameters parameters); void addField(String f); void reset(); @Override void tokenize(Document document); ArrayList<IntSpan> getTokenPositions(); static final Logger log; static HashSet<String> ignoredTags; public TagTokenizerParser state; }
@Test public void testTokenize() throws IOException, IncompatibleProcessorException { TagTokenizer tokenizer = new TagTokenizer(); tokenizer.addField("title"); tokenizer.addField("a"); tokenizer.addField("html"); Document document = new Document(); tokenizer.setProcessor(new NullProcessor(Document.class)); document.text = "<html> <a href=\"http: + "<title>title text</title> \n" + "That's all folks!" + "</html>" + "ciir.cs.umass.edu " + "m.b.a. " + "17.2"; tokenizer.process(document); String[] tokens = {"yahoo", "this", "is", "some", "text", "title", "text", "thats", "all", "folks", "ciir", "cs", "umass", "edu", "mba", "17", "2"}; assertEquals("Token length", tokens.length, document.terms.size()); for (int i = 0; i < tokens.length; i++) { assertEquals("Token text", tokens[i], document.terms.get(i)); } assertEquals(3, document.tags.size()); Tag html = document.tags.get(0); assertEquals(html.name, "html"); assertEquals(html.attributes.size(), 0); assertEquals(html.begin, 0); assertEquals(html.end, 10); Tag a = document.tags.get(1); assertEquals(a.name, "a"); assertEquals(a.attributes.size(), 1); assertEquals(a.attributes.get("href"), "http: assertEquals(a.begin, 0); assertEquals(a.end, 1); Tag title = document.tags.get(2); assertEquals(title.name, "title"); assertEquals(title.attributes.size(), 0); assertEquals(title.begin, 5); assertEquals(title.end, 7); }