@Test public void testParse() throws IOException { Map<String, String> typeToClassMap = new HashMap<>(); BratAnnotationStreamTest.addEntityTypes(typeToClassMap); AnnotationConfiguration config = new AnnotationConfiguration(typeToClassMap); InputStream txtIn = BratDocumentTest.class.getResourceAsStream( "/opennlp/tools/formats/brat/opennlp-1193.txt"); InputStream annIn = BratDocumentTest.class.getResourceAsStream( "/opennlp/tools/formats/brat/opennlp-1193.ann"); BratDocument doc = BratDocument.parseDocument(config, "opennlp-1193", txtIn, annIn); BratDocumentParser parser = new BratDocumentParser(new NewlineSentenceDetector(), WhitespaceTokenizer.INSTANCE); List<NameSample> names = parser.parse(doc); Assert.assertEquals(3, names.size()); NameSample sample1 = names.get(0); Assert.assertEquals(1, sample1.getNames().length); Assert.assertEquals(0, sample1.getNames()[0].getStart()); Assert.assertEquals(2, sample1.getNames()[0].getEnd()); NameSample sample2 = names.get(1); Assert.assertEquals(1, sample2.getNames().length); Assert.assertEquals(0, sample2.getNames()[0].getStart()); Assert.assertEquals(1, sample2.getNames()[0].getEnd()); NameSample sample3 = names.get(2); Assert.assertEquals(3, sample3.getNames().length); Assert.assertEquals(0, sample3.getNames()[0].getStart()); Assert.assertEquals(1, sample3.getNames()[0].getEnd()); Assert.assertEquals(1, sample3.getNames()[1].getStart()); Assert.assertEquals(2, sample3.getNames()[1].getEnd()); Assert.assertEquals(2, sample3.getNames()[2].getStart()); Assert.assertEquals(3, sample3.getNames()[2].getEnd()); }
public List<NameSample> parse(BratDocument sample) { Set<String> entityIdSet = new HashSet<>(); Map<Integer, Span> coveredIndexes = new HashMap<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { entityIdSet.add(ann.getId()); for (Span span : ((SpanAnnotation) ann).getSpans()) { for (int i = span.getStart(); i < span.getEnd(); i++) { coveredIndexes.put(i, span); } } } } List<Span> sentences = new ArrayList<>(); for (Span sentence : sentDetector.sentPosDetect(sample.getText())) { Span conflictingName = coveredIndexes.get(sentence.getStart()); if (sentences.size() > 0 && conflictingName != null && conflictingName.getStart() < sentence.getStart()) { Span lastSentence = sentences.remove(sentences.size() - 1); sentences.add(new Span(lastSentence.getStart(), sentence.getEnd())); System.out.println("Correcting sentence segmentation in document " + sample.getId()); } else { sentences.add(sentence); } } List<NameSample> samples = new ArrayList<>(sentences.size()); for (Span sentence : sentences) { String sentenceText = sentence.getCoveredText( sample.getText()).toString(); Span[] tokens = tokenizer.tokenizePos(sentenceText); Map<Integer, Integer> tokenIndexMap = new HashMap<>(); for (int i = 0; i < tokens.length; i++) { tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i); tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1); } List<Span> names = new ArrayList<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { SpanAnnotation entity = (SpanAnnotation) ann; List<Span> mappedFragments = new ArrayList<>(); for (Span entitySpan : entity.getSpans()) { if (sentence.contains(entitySpan)) { entityIdSet.remove(ann.getId()); entitySpan = entitySpan.trim(sample.getText()); Integer nameBeginIndex = tokenIndexMap.get(-entitySpan.getStart()); Integer nameEndIndex = tokenIndexMap.get(entitySpan.getEnd()); if (nameBeginIndex != null && nameEndIndex != null) { mappedFragments.add(new Span(nameBeginIndex, nameEndIndex, entity.getType())); } else { System.err.println("Dropped entity " + entity.getId() + " (" + entitySpan.getCoveredText(sample.getText()) + ") " + " in document " + sample.getId() + ", it is not matching tokenization!"); } } } Collections.sort(mappedFragments); for (int i = 1; i < mappedFragments.size(); i++) { if (mappedFragments.get(i - 1).getEnd() == mappedFragments.get(i).getStart()) { mappedFragments.set(i, new Span(mappedFragments.get(i - 1).getStart(), mappedFragments.get(i).getEnd(), mappedFragments.get(i).getType())); mappedFragments.set(i - 1, null); } } for (Span span : mappedFragments) { if (span != null ) { names.add(span); } } } } samples.add(new NameSample(sample.getId(), Span.spansToStrings(tokens, sentenceText), names.toArray(new Span[names.size()]), null, samples.size() == 0)); } for (String id : entityIdSet) { System.err.println("Dropped entity " + id + " in document " + sample.getId() + ", is not matching sentence segmentation!"); } return samples; }
BratDocumentParser { public List<NameSample> parse(BratDocument sample) { Set<String> entityIdSet = new HashSet<>(); Map<Integer, Span> coveredIndexes = new HashMap<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { entityIdSet.add(ann.getId()); for (Span span : ((SpanAnnotation) ann).getSpans()) { for (int i = span.getStart(); i < span.getEnd(); i++) { coveredIndexes.put(i, span); } } } } List<Span> sentences = new ArrayList<>(); for (Span sentence : sentDetector.sentPosDetect(sample.getText())) { Span conflictingName = coveredIndexes.get(sentence.getStart()); if (sentences.size() > 0 && conflictingName != null && conflictingName.getStart() < sentence.getStart()) { Span lastSentence = sentences.remove(sentences.size() - 1); sentences.add(new Span(lastSentence.getStart(), sentence.getEnd())); System.out.println("Correcting sentence segmentation in document " + sample.getId()); } else { sentences.add(sentence); } } List<NameSample> samples = new ArrayList<>(sentences.size()); for (Span sentence : sentences) { String sentenceText = sentence.getCoveredText( sample.getText()).toString(); Span[] tokens = tokenizer.tokenizePos(sentenceText); Map<Integer, Integer> tokenIndexMap = new HashMap<>(); for (int i = 0; i < tokens.length; i++) { tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i); tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1); } List<Span> names = new ArrayList<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { SpanAnnotation entity = (SpanAnnotation) ann; List<Span> mappedFragments = new ArrayList<>(); for (Span entitySpan : entity.getSpans()) { if (sentence.contains(entitySpan)) { entityIdSet.remove(ann.getId()); entitySpan = entitySpan.trim(sample.getText()); Integer nameBeginIndex = tokenIndexMap.get(-entitySpan.getStart()); Integer nameEndIndex = tokenIndexMap.get(entitySpan.getEnd()); if (nameBeginIndex != null && nameEndIndex != null) { mappedFragments.add(new Span(nameBeginIndex, nameEndIndex, entity.getType())); } else { System.err.println("Dropped entity " + entity.getId() + " (" + entitySpan.getCoveredText(sample.getText()) + ") " + " in document " + sample.getId() + ", it is not matching tokenization!"); } } } Collections.sort(mappedFragments); for (int i = 1; i < mappedFragments.size(); i++) { if (mappedFragments.get(i - 1).getEnd() == mappedFragments.get(i).getStart()) { mappedFragments.set(i, new Span(mappedFragments.get(i - 1).getStart(), mappedFragments.get(i).getEnd(), mappedFragments.get(i).getType())); mappedFragments.set(i - 1, null); } } for (Span span : mappedFragments) { if (span != null ) { names.add(span); } } } } samples.add(new NameSample(sample.getId(), Span.spansToStrings(tokens, sentenceText), names.toArray(new Span[names.size()]), null, samples.size() == 0)); } for (String id : entityIdSet) { System.err.println("Dropped entity " + id + " in document " + sample.getId() + ", is not matching sentence segmentation!"); } return samples; } }
BratDocumentParser { public List<NameSample> parse(BratDocument sample) { Set<String> entityIdSet = new HashSet<>(); Map<Integer, Span> coveredIndexes = new HashMap<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { entityIdSet.add(ann.getId()); for (Span span : ((SpanAnnotation) ann).getSpans()) { for (int i = span.getStart(); i < span.getEnd(); i++) { coveredIndexes.put(i, span); } } } } List<Span> sentences = new ArrayList<>(); for (Span sentence : sentDetector.sentPosDetect(sample.getText())) { Span conflictingName = coveredIndexes.get(sentence.getStart()); if (sentences.size() > 0 && conflictingName != null && conflictingName.getStart() < sentence.getStart()) { Span lastSentence = sentences.remove(sentences.size() - 1); sentences.add(new Span(lastSentence.getStart(), sentence.getEnd())); System.out.println("Correcting sentence segmentation in document " + sample.getId()); } else { sentences.add(sentence); } } List<NameSample> samples = new ArrayList<>(sentences.size()); for (Span sentence : sentences) { String sentenceText = sentence.getCoveredText( sample.getText()).toString(); Span[] tokens = tokenizer.tokenizePos(sentenceText); Map<Integer, Integer> tokenIndexMap = new HashMap<>(); for (int i = 0; i < tokens.length; i++) { tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i); tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1); } List<Span> names = new ArrayList<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { SpanAnnotation entity = (SpanAnnotation) ann; List<Span> mappedFragments = new ArrayList<>(); for (Span entitySpan : entity.getSpans()) { if (sentence.contains(entitySpan)) { entityIdSet.remove(ann.getId()); entitySpan = entitySpan.trim(sample.getText()); Integer nameBeginIndex = tokenIndexMap.get(-entitySpan.getStart()); Integer nameEndIndex = tokenIndexMap.get(entitySpan.getEnd()); if (nameBeginIndex != null && nameEndIndex != null) { mappedFragments.add(new Span(nameBeginIndex, nameEndIndex, entity.getType())); } else { System.err.println("Dropped entity " + entity.getId() + " (" + entitySpan.getCoveredText(sample.getText()) + ") " + " in document " + sample.getId() + ", it is not matching tokenization!"); } } } Collections.sort(mappedFragments); for (int i = 1; i < mappedFragments.size(); i++) { if (mappedFragments.get(i - 1).getEnd() == mappedFragments.get(i).getStart()) { mappedFragments.set(i, new Span(mappedFragments.get(i - 1).getStart(), mappedFragments.get(i).getEnd(), mappedFragments.get(i).getType())); mappedFragments.set(i - 1, null); } } for (Span span : mappedFragments) { if (span != null ) { names.add(span); } } } } samples.add(new NameSample(sample.getId(), Span.spansToStrings(tokens, sentenceText), names.toArray(new Span[names.size()]), null, samples.size() == 0)); } for (String id : entityIdSet) { System.err.println("Dropped entity " + id + " in document " + sample.getId() + ", is not matching sentence segmentation!"); } return samples; } BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer);  BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer,
                            Set<String> nameTypes); }
BratDocumentParser { public List<NameSample> parse(BratDocument sample) { Set<String> entityIdSet = new HashSet<>(); Map<Integer, Span> coveredIndexes = new HashMap<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { entityIdSet.add(ann.getId()); for (Span span : ((SpanAnnotation) ann).getSpans()) { for (int i = span.getStart(); i < span.getEnd(); i++) { coveredIndexes.put(i, span); } } } } List<Span> sentences = new ArrayList<>(); for (Span sentence : sentDetector.sentPosDetect(sample.getText())) { Span conflictingName = coveredIndexes.get(sentence.getStart()); if (sentences.size() > 0 && conflictingName != null && conflictingName.getStart() < sentence.getStart()) { Span lastSentence = sentences.remove(sentences.size() - 1); sentences.add(new Span(lastSentence.getStart(), sentence.getEnd())); System.out.println("Correcting sentence segmentation in document " + sample.getId()); } else { sentences.add(sentence); } } List<NameSample> samples = new ArrayList<>(sentences.size()); for (Span sentence : sentences) { String sentenceText = sentence.getCoveredText( sample.getText()).toString(); Span[] tokens = tokenizer.tokenizePos(sentenceText); Map<Integer, Integer> tokenIndexMap = new HashMap<>(); for (int i = 0; i < tokens.length; i++) { tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i); tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1); } List<Span> names = new ArrayList<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { SpanAnnotation entity = (SpanAnnotation) ann; List<Span> mappedFragments = new ArrayList<>(); for (Span entitySpan : entity.getSpans()) { if (sentence.contains(entitySpan)) { entityIdSet.remove(ann.getId()); entitySpan = entitySpan.trim(sample.getText()); Integer nameBeginIndex = tokenIndexMap.get(-entitySpan.getStart()); Integer nameEndIndex = tokenIndexMap.get(entitySpan.getEnd()); if (nameBeginIndex != null && nameEndIndex != null) { mappedFragments.add(new Span(nameBeginIndex, nameEndIndex, entity.getType())); } else { System.err.println("Dropped entity " + entity.getId() + " (" + entitySpan.getCoveredText(sample.getText()) + ") " + " in document " + sample.getId() + ", it is not matching tokenization!"); } } } Collections.sort(mappedFragments); for (int i = 1; i < mappedFragments.size(); i++) { if (mappedFragments.get(i - 1).getEnd() == mappedFragments.get(i).getStart()) { mappedFragments.set(i, new Span(mappedFragments.get(i - 1).getStart(), mappedFragments.get(i).getEnd(), mappedFragments.get(i).getType())); mappedFragments.set(i - 1, null); } } for (Span span : mappedFragments) { if (span != null ) { names.add(span); } } } } samples.add(new NameSample(sample.getId(), Span.spansToStrings(tokens, sentenceText), names.toArray(new Span[names.size()]), null, samples.size() == 0)); } for (String id : entityIdSet) { System.err.println("Dropped entity " + id + " in document " + sample.getId() + ", is not matching sentence segmentation!"); } return samples; } BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer);  BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer,
                            Set<String> nameTypes); List<NameSample> parse(BratDocument sample); }
BratDocumentParser { public List<NameSample> parse(BratDocument sample) { Set<String> entityIdSet = new HashSet<>(); Map<Integer, Span> coveredIndexes = new HashMap<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { entityIdSet.add(ann.getId()); for (Span span : ((SpanAnnotation) ann).getSpans()) { for (int i = span.getStart(); i < span.getEnd(); i++) { coveredIndexes.put(i, span); } } } } List<Span> sentences = new ArrayList<>(); for (Span sentence : sentDetector.sentPosDetect(sample.getText())) { Span conflictingName = coveredIndexes.get(sentence.getStart()); if (sentences.size() > 0 && conflictingName != null && conflictingName.getStart() < sentence.getStart()) { Span lastSentence = sentences.remove(sentences.size() - 1); sentences.add(new Span(lastSentence.getStart(), sentence.getEnd())); System.out.println("Correcting sentence segmentation in document " + sample.getId()); } else { sentences.add(sentence); } } List<NameSample> samples = new ArrayList<>(sentences.size()); for (Span sentence : sentences) { String sentenceText = sentence.getCoveredText( sample.getText()).toString(); Span[] tokens = tokenizer.tokenizePos(sentenceText); Map<Integer, Integer> tokenIndexMap = new HashMap<>(); for (int i = 0; i < tokens.length; i++) { tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i); tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1); } List<Span> names = new ArrayList<>(); for (BratAnnotation ann : sample.getAnnotations()) { if (isSpanAnnotation(ann)) { SpanAnnotation entity = (SpanAnnotation) ann; List<Span> mappedFragments = new ArrayList<>(); for (Span entitySpan : entity.getSpans()) { if (sentence.contains(entitySpan)) { entityIdSet.remove(ann.getId()); entitySpan = entitySpan.trim(sample.getText()); Integer nameBeginIndex = tokenIndexMap.get(-entitySpan.getStart()); Integer nameEndIndex = tokenIndexMap.get(entitySpan.getEnd()); if (nameBeginIndex != null && nameEndIndex != null) { mappedFragments.add(new Span(nameBeginIndex, nameEndIndex, entity.getType())); } else { System.err.println("Dropped entity " + entity.getId() + " (" + entitySpan.getCoveredText(sample.getText()) + ") " + " in document " + sample.getId() + ", it is not matching tokenization!"); } } } Collections.sort(mappedFragments); for (int i = 1; i < mappedFragments.size(); i++) { if (mappedFragments.get(i - 1).getEnd() == mappedFragments.get(i).getStart()) { mappedFragments.set(i, new Span(mappedFragments.get(i - 1).getStart(), mappedFragments.get(i).getEnd(), mappedFragments.get(i).getType())); mappedFragments.set(i - 1, null); } } for (Span span : mappedFragments) { if (span != null ) { names.add(span); } } } } samples.add(new NameSample(sample.getId(), Span.spansToStrings(tokens, sentenceText), names.toArray(new Span[names.size()]), null, samples.size() == 0)); } for (String id : entityIdSet) { System.err.println("Dropped entity " + id + " in document " + sample.getId() + ", is not matching sentence segmentation!"); } return samples; } BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer);  BratDocumentParser(SentenceDetector sentenceDetector, Tokenizer tokenizer,
                            Set<String> nameTypes); List<NameSample> parse(BratDocument sample);  }