Conll03NameSampleStream implements ObjectStream<NameSample> { public NameSample read() throws IOException { List<String> sentence = new ArrayList<String>(); List<String> tags = new ArrayList<String>(); boolean isClearAdaptiveData = false; String line; while ((line = lineStream.read()) != null && !StringUtil.isEmpty(line)) { if (line.startsWith(Conll02NameSampleStream.DOCSTART)) { isClearAdaptiveData = true; String emptyLine = lineStream.read(); if (!StringUtil.isEmpty(emptyLine)) throw new IOException("Empty line after -DOCSTART- not empty: '" + emptyLine +"'!"); continue; } String fields[] = line.split(" "); if (LANGUAGE.EN.equals(lang) && (fields.length == 4)) { sentence.add(fields[0]); tags.add(fields[3]); } else if (LANGUAGE.DE.equals(lang) && (fields.length == 5)) { sentence.add(fields[0]); tags.add(fields[4]); } else { throw new IOException("Incorrect number of fields per line for language: '" + line + "'!"); } } if (sentence.size() > 0) { List<Span> names = new ArrayList<Span>(); int beginIndex = -1; int endIndex = -1; for (int i = 0; i < tags.size(); i++) { String tag = tags.get(i); if (tag.endsWith("PER") && (types & Conll02NameSampleStream.GENERATE_PERSON_ENTITIES) == 0) tag = "O"; if (tag.endsWith("ORG") && (types & Conll02NameSampleStream.GENERATE_ORGANIZATION_ENTITIES) == 0) tag = "O"; if (tag.endsWith("LOC") && (types & Conll02NameSampleStream.GENERATE_LOCATION_ENTITIES) == 0) tag = "O"; if (tag.endsWith("MISC") && (types & Conll02NameSampleStream.GENERATE_MISC_ENTITIES) == 0) tag = "O"; if (tag.equals("O")) { if (beginIndex != -1) { names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); beginIndex = -1; endIndex = -1; } } else if (tag.startsWith("B-")) { if (beginIndex != -1) { names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); } beginIndex = i; endIndex = i + 1; } else if (tag.startsWith("I-")) { if (beginIndex == -1) { beginIndex = i; endIndex = i + 1; } else if (!tag.endsWith(tags.get(beginIndex).substring(1))) { names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); beginIndex = i; endIndex = i + 1; } else { endIndex ++; } } else { throw new IOException("Invalid tag: " + tag); } } if (beginIndex != -1) names.add(extract(beginIndex, endIndex, tags.get(beginIndex))); return new NameSample(sentence.toArray(new String[sentence.size()]), names.toArray(new Span[names.size()]), isClearAdaptiveData); } else if (line != null) { return read(); } else { return null; } } Conll03NameSampleStream(LANGUAGE lang, ObjectStream<String> lineStream, int types);  Conll03NameSampleStream(LANGUAGE lang, InputStream in, int types); NameSample read(); void reset(); void close();  }
@Test public void testParsingEnglishSample() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.EN, ENGLISH_SAMPLE); NameSample personName = sampleStream.read(); assertNotNull(personName); assertEquals(9, personName.getSentence().length); assertEquals(0, personName.getNames().length); assertEquals(true, personName.isClearAdaptiveDataSet()); personName = sampleStream.read(); assertNotNull(personName); assertEquals(2, personName.getSentence().length); assertEquals(1, personName.getNames().length); assertEquals(false, personName.isClearAdaptiveDataSet()); Span nameSpan = personName.getNames()[0]; assertEquals(0, nameSpan.getStart()); assertEquals(2, nameSpan.getEnd()); assertNull(sampleStream.read()); }
@Test(expected=IOException.class) public void testParsingEnglishSampleWithGermanAsLanguage() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.DE, ENGLISH_SAMPLE); sampleStream.read(); }
@Test(expected=IOException.class) public void testParsingGermanSampleWithEnglishAsLanguage() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.EN, GERMAN_SAMPLE); sampleStream.read(); }
@Test public void testParsingGermanSample() throws IOException { ObjectStream<NameSample> sampleStream = openData(LANGUAGE.DE, GERMAN_SAMPLE); NameSample personName = sampleStream.read(); assertNotNull(personName); assertEquals(5, personName.getSentence().length); assertEquals(0, personName.getNames().length); assertEquals(true, personName.isClearAdaptiveDataSet()); }