Conll02NameSampleStream implements ObjectStream<NameSample> { public NameSample read() throws IOException { List<String> sentence = new ArrayList<String>(); List<String> tags = new ArrayList<String>(); boolean isClearAdaptiveData = false; String line; while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line)) { if (LANGUAGE.NL.equals(lang) && line.startsWith(DOCSTART)) { isClearAdaptiveData = true; continue; } String fields[] = line.split(" "); if (fields.length == 3) { sentence.add(fields[0]); tags.add(fields[2]); } else { throw new IOException("Expected three fields per line in training data, got " + fields.length + " for line '" + line + "'!"); } } if (LANGUAGE.ES.equals(lang)) isClearAdaptiveData = true; if (sentence.size() > 0) { List<Span> names = new ArrayList<Span>(); int beginIndex = -1; int endIndex = -1; for (int i = 0; i < tags.size(); i++) { String tag = tags.get(i); if (tag.endsWith("PER") && (types & GENERATE_PERSON_ENTITIES) == 0) tag = "O"; if (tag.endsWith("ORG") && (types & GENERATE_ORGANIZATION_ENTITIES) == 0) tag = "O"; if (tag.endsWith("LOC") && (types & GENERATE_LOCATION_ENTITIES) == 0) tag = "O"; if (tag.endsWith("MISC") && (types & GENERATE_MISC_ENTITIES) == 0) tag = "O"; if (tag.startsWith("B-")) { if (beginIndex != -1) { names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); beginIndex = -1; endIndex = -1; } beginIndex = i; endIndex = i +1; } else if (tag.startsWith("I-")) { endIndex++; } else if (tag.equals("O")) { if (beginIndex != -1) { names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); beginIndex = -1; endIndex = -1; } } else { throw new IOException("Invalid tag: " + tag); } } if (beginIndex != -1) names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); return new NameSample(sentence.toArray(new String[sentence.size()]), names.toArray(new Span[names.size()]), isClearAdaptiveData); } else if (line != null) { return read(); } else { return null; } } Conll02NameSampleStream(LANGUAGE lang, ObjectStream<String> lineStream, int types);  Conll02NameSampleStream(LANGUAGE lang, InputStream in, int types); NameSample read(); void reset(); void close(); static final int GENERATE_PERSON_ENTITIES; static final int GENERATE_ORGANIZATION_ENTITIES; static final int GENERATE_LOCATION_ENTITIES; static final int GENERATE_MISC_ENTITIES; static final String DOCSTART; }
@Test public void testParsingSpanishSample() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.ES, "conll2002-es.sample"); NameSample personName = sampleStream.read(); assertNotNull(personName); assertEquals(5, personName.getSentence().length); assertEquals(1, personName.getNames().length); assertEquals(true, personName.isClearAdaptiveDataSet()); Span nameSpan = personName.getNames()[0]; assertEquals(0, nameSpan.getStart()); assertEquals(4, nameSpan.getEnd()); assertEquals(true, personName.isClearAdaptiveDataSet()); assertEquals(0, sampleStream.read().getNames().length); assertNull(sampleStream.read()); }
@Test public void testParsingDutchSample() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.NL, "conll2002-nl.sample"); NameSample personName = sampleStream.read(); assertEquals(0, personName.getNames().length); assertTrue(personName.isClearAdaptiveDataSet()); personName = sampleStream.read(); assertFalse(personName.isClearAdaptiveDataSet()); assertNull(sampleStream.read()); }