@Test public void testHtmlNameSampleParsing() throws IOException { InputStreamFactory in = new ResourceAsStreamFactory(getClass(), "/opennlp/tools/namefind/html1.train"); NameSampleDataStream ds = new NameSampleDataStream( new PlainTextByLineStream(in, StandardCharsets.UTF_8)); NameSample ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("<html>", ns.getSentence()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("<head/>", ns.getSentence()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("<body>", ns.getSentence()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("<ul>", ns.getSentence()[0]); ns = ds.read(); assertEquals(6, ns.getSentence().length); assertEquals("<li>", ns.getSentence()[0]); assertEquals("Advanced", ns.getSentence()[1]); assertEquals("Integrated", ns.getSentence()[2]); assertEquals("Pest", ns.getSentence()[3]); assertEquals("Management", ns.getSentence()[4]); assertEquals("</li>", ns.getSentence()[5]); assertEquals(new Span(1, 5, organization), ns.getNames()[0]); ns = ds.read(); assertEquals(7, ns.getSentence().length); assertEquals("<li>", ns.getSentence()[0]); assertEquals("Bay", ns.getSentence()[1]); assertEquals("Cities", ns.getSentence()[2]); assertEquals("Produce", ns.getSentence()[3]); assertEquals("Co.,", ns.getSentence()[4]); assertEquals("Inc.", ns.getSentence()[5]); assertEquals("</li>", ns.getSentence()[6]); assertEquals(new Span(1, 6, organization), ns.getNames()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("</ul>", ns.getSentence()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("</body>", ns.getSentence()[0]); ns = ds.read(); assertEquals(1, ns.getSentence().length); assertEquals("</html>", ns.getSentence()[0]); assertNull(ds.read()); ds.close(); }
public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); NameSample read(); }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); NameSample read(); static final String START_TAG_PREFIX; static final String START_TAG; static final String END_TAG; }