@Test public void testWithNameTypes() throws Exception { InputStreamFactory in = new ResourceAsStreamFactory(getClass(), "/opennlp/tools/namefind/voa1.train"); NameSampleDataStream ds = new NameSampleDataStream( new PlainTextByLineStream(in, StandardCharsets.UTF_8)); Map<String, List<String>> names = new HashMap<>(); Map<String, List<Span>> spans = new HashMap<>(); NameSample ns; while ((ns = ds.read()) != null) { Span[] nameSpans = ns.getNames(); for (Span nameSpan : nameSpans) { if (!names.containsKey(nameSpan.getType())) { names.put(nameSpan.getType(), new ArrayList<>()); spans.put(nameSpan.getType(), new ArrayList<>()); } names.get(nameSpan.getType()).add(sublistToString(ns.getSentence(), nameSpan)); spans.get(nameSpan.getType()).add(nameSpan); } } ds.close(); String[] expectedPerson = { "Barack Obama", "Obama", "Obama", "Lee Myung - bak", "Obama", "Obama", "Scott Snyder", "Snyder", "Obama", "Obama", "Obama", "Tim Peters", "Obama", "Peters" }; String[] expectedDate = { "Wednesday", "Thursday", "Wednesday" }; String[] expectedLocation = { "U . S .", "South Korea", "North Korea", "China", "South Korea", "North Korea", "North Korea", "U . S .", "South Korea", "United States", "Pyongyang", "North Korea", "South Korea", "Afghanistan", "Seoul", "U . S .", "China" }; String[] expectedOrganization = {"Center for U . S . Korea Policy"}; assertEquals(expectedPerson.length, names.get(person).size()); assertEquals(expectedDate.length, names.get(date).size()); assertEquals(expectedLocation.length, names.get(location).size()); assertEquals(expectedOrganization.length, names.get(organization).size()); assertEquals(new Span(5,7, person), spans.get(person).get(0)); assertEquals(expectedPerson[0], names.get(person).get(0)); assertEquals(new Span(10,11, person), spans.get(person).get(1)); assertEquals(expectedPerson[1], names.get(person).get(1)); assertEquals(new Span(29,30, person), spans.get(person).get(2)); assertEquals(expectedPerson[2], names.get(person).get(2)); assertEquals(new Span(23,27, person), spans.get(person).get(3)); assertEquals(expectedPerson[3], names.get(person).get(3)); assertEquals(new Span(1,2, person), spans.get(person).get(4)); assertEquals(expectedPerson[4], names.get(person).get(4)); assertEquals(new Span(8,9, person), spans.get(person).get(5)); assertEquals(expectedPerson[5], names.get(person).get(5)); assertEquals(new Span(0,2, person), spans.get(person).get(6)); assertEquals(expectedPerson[6], names.get(person).get(6)); assertEquals(new Span(25,26, person), spans.get(person).get(7)); assertEquals(expectedPerson[7], names.get(person).get(7)); assertEquals(new Span(1,2, person), spans.get(person).get(8)); assertEquals(expectedPerson[8], names.get(person).get(8)); assertEquals(new Span(6,7, person), spans.get(person).get(9)); assertEquals(expectedPerson[9], names.get(person).get(9)); assertEquals(new Span(14,15, person), spans.get(person).get(10)); assertEquals(expectedPerson[10], names.get(person).get(10)); assertEquals(new Span(0,2, person), spans.get(person).get(11)); assertEquals(expectedPerson[11], names.get(person).get(11)); assertEquals(new Span(12,13, person), spans.get(person).get(12)); assertEquals(expectedPerson[12], names.get(person).get(12)); assertEquals(new Span(12,13, person), spans.get(person).get(13)); assertEquals(expectedPerson[13], names.get(person).get(13)); assertEquals(new Span(7,8, date), spans.get(date).get(0)); assertEquals(expectedDate[0], names.get(date).get(0)); assertEquals(new Span(27,28, date), spans.get(date).get(1)); assertEquals(expectedDate[1], names.get(date).get(1)); assertEquals(new Span(15,16, date), spans.get(date).get(2)); assertEquals(expectedDate[2], names.get(date).get(2)); assertEquals(new Span(0, 4, location), spans.get(location).get(0)); assertEquals(expectedLocation[0], names.get(location).get(0)); assertEquals(new Span(10,12, location), spans.get(location).get(1)); assertEquals(expectedLocation[1], names.get(location).get(1)); assertEquals(new Span(28,30, location), spans.get(location).get(2)); assertEquals(expectedLocation[2], names.get(location).get(2)); assertEquals(new Span(3,4, location), spans.get(location).get(3)); assertEquals(expectedLocation[3], names.get(location).get(3)); assertEquals(new Span(5,7, location), spans.get(location).get(4)); assertEquals(expectedLocation[4], names.get(location).get(4)); assertEquals(new Span(16,18, location), spans.get(location).get(5)); assertEquals(expectedLocation[5], names.get(location).get(5)); assertEquals(new Span(1,3, location), spans.get(location).get(6)); assertEquals(expectedLocation[6], names.get(location).get(6)); assertEquals(new Span(5,9, location), spans.get(location).get(7)); assertEquals(expectedLocation[7], names.get(location).get(7)); assertEquals(new Span(0,2, location), spans.get(location).get(8)); assertEquals(expectedLocation[8], names.get(location).get(8)); assertEquals(new Span(4,6, location), spans.get(location).get(9)); assertEquals(expectedLocation[9], names.get(location).get(9)); assertEquals(new Span(10,11, location), spans.get(location).get(10)); assertEquals(expectedLocation[10], names.get(location).get(10)); assertEquals(new Span(6,8, location), spans.get(location).get(11)); assertEquals(expectedLocation[11], names.get(location).get(11)); assertEquals(new Span(4,6, location), spans.get(location).get(12)); assertEquals(expectedLocation[12], names.get(location).get(12)); assertEquals(new Span(10,11, location), spans.get(location).get(13)); assertEquals(expectedLocation[13], names.get(location).get(13)); assertEquals(new Span(12,13, location), spans.get(location).get(14)); assertEquals(expectedLocation[14], names.get(location).get(14)); assertEquals(new Span(5,9, location), spans.get(location).get(15)); assertEquals(expectedLocation[15], names.get(location).get(15)); assertEquals(new Span(11,12, location), spans.get(location).get(16)); assertEquals(expectedLocation[16], names.get(location).get(16)); assertEquals(new Span(7,15, organization), spans.get(organization).get(0)); assertEquals(expectedOrganization[0], names.get(organization).get(0)); }
public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); NameSample read(); }
NameSampleDataStream extends FilterObjectStream<String, NameSample> { public NameSample read() throws IOException { String token = samples.read(); boolean isClearAdaptiveData = false; while (token != null && token.trim().length() == 0) { isClearAdaptiveData = true; token = samples.read(); } if (token != null) { return NameSample.parse(token, isClearAdaptiveData); } else { return null; } } NameSampleDataStream(ObjectStream<String> in); NameSample read(); static final String START_TAG_PREFIX; static final String START_TAG; static final String END_TAG; }