@Test public void testDocument0Reader() throws IOException { Resource file = Resource.fromJAR("testdata/222.0"); Document doc = new MultiSegReader() .readDocumentFromFile(file); assertEquals("222.0", doc.getId()); List<SectionAnnotation> anns = doc.streamAnnotations(Annotation.Source.GOLD, SectionAnnotation.class).sorted().collect(Collectors.toList()); for(int i = 0; i < 11; i++) { assertEquals(Integer.toString(i), anns.get(i).getSectionHeading()); assertEquals(doc.getSentence(i).getBegin(), anns.get(i).getBegin()); assertEquals(doc.getSentence(i).getEnd(), anns.get(i).getEnd()); } assertEquals("I finally decided to try alternative medicine.", doc.getSentenceAtPosition(anns.get(2).getBegin()).get().getText()); assertEquals("I'm a little nervous about trying these unorthodox treatments, but after hearing about Amelia's and Ray's experiences, I'm willing to give them a try. Nothing has worked so far, so what do I have to lose?", doc.getSentence(10).getText()); assertEquals(11, doc.countSentences()); assertEquals(11, doc.countAnnotations()); }
@Override public Document readDocumentFromFile(Resource file) { try(InputStream in = file.getInputStream()) { CharsetDecoder utf8 = StandardCharsets.UTF_8.newDecoder(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); Iterator<String> it = new LineIterator(br); int i = 0; int n = 0; Document doc = new Document(); doc.setId(file.getFileName()); doc.setSource(file.toString()); doc.setType("multiseg"); String line; StringBuilder text = new StringBuilder(); Pattern filePattern = Pattern.compile("^(.+?/)(.+?)\\.(\\d+)$"); Matcher m = filePattern.matcher(file.toString()); if(!m.matches()) throw new IllegalArgumentException("invalid file name"); String basePath = m.group(1); String baseDoc = m.group(2); int docNum = Integer.parseInt(m.group(3)); Resource labels = file instanceof InternalResource ? Resource.fromJAR(basePath + baseDoc + ".label") : Resource.fromFile(basePath, baseDoc + ".label"); TreeSet[] sections = readSectionsFromLabel(labels, docNum); while(it.hasNext()) { line = it.next(); if(sections[0].contains(i)) { text = new StringBuilder(); } text.append(line).append("\n"); if(sections[1].contains(i)) { String sectionText = text.toString(); if(sectionText.trim().length() > 0) { addToDocument(sectionText, n++, doc); } text = new StringBuilder(); } i++; } return doc; } catch(IOException ex) { log.error(ex.toString()); throw new RuntimeException(ex.toString(), ex.getCause()); } }
MultiSegReader extends RawTextDatasetReader { @Override public Document readDocumentFromFile(Resource file) { try(InputStream in = file.getInputStream()) { CharsetDecoder utf8 = StandardCharsets.UTF_8.newDecoder(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); Iterator<String> it = new LineIterator(br); int i = 0; int n = 0; Document doc = new Document(); doc.setId(file.getFileName()); doc.setSource(file.toString()); doc.setType("multiseg"); String line; StringBuilder text = new StringBuilder(); Pattern filePattern = Pattern.compile("^(.+?/)(.+?)\\.(\\d+)$"); Matcher m = filePattern.matcher(file.toString()); if(!m.matches()) throw new IllegalArgumentException("invalid file name"); String basePath = m.group(1); String baseDoc = m.group(2); int docNum = Integer.parseInt(m.group(3)); Resource labels = file instanceof InternalResource ? Resource.fromJAR(basePath + baseDoc + ".label") : Resource.fromFile(basePath, baseDoc + ".label"); TreeSet[] sections = readSectionsFromLabel(labels, docNum); while(it.hasNext()) { line = it.next(); if(sections[0].contains(i)) { text = new StringBuilder(); } text.append(line).append("\n"); if(sections[1].contains(i)) { String sectionText = text.toString(); if(sectionText.trim().length() > 0) { addToDocument(sectionText, n++, doc); } text = new StringBuilder(); } i++; } return doc; } catch(IOException ex) { log.error(ex.toString()); throw new RuntimeException(ex.toString(), ex.getCause()); } } }
MultiSegReader extends RawTextDatasetReader { @Override public Document readDocumentFromFile(Resource file) { try(InputStream in = file.getInputStream()) { CharsetDecoder utf8 = StandardCharsets.UTF_8.newDecoder(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); Iterator<String> it = new LineIterator(br); int i = 0; int n = 0; Document doc = new Document(); doc.setId(file.getFileName()); doc.setSource(file.toString()); doc.setType("multiseg"); String line; StringBuilder text = new StringBuilder(); Pattern filePattern = Pattern.compile("^(.+?/)(.+?)\\.(\\d+)$"); Matcher m = filePattern.matcher(file.toString()); if(!m.matches()) throw new IllegalArgumentException("invalid file name"); String basePath = m.group(1); String baseDoc = m.group(2); int docNum = Integer.parseInt(m.group(3)); Resource labels = file instanceof InternalResource ? Resource.fromJAR(basePath + baseDoc + ".label") : Resource.fromFile(basePath, baseDoc + ".label"); TreeSet[] sections = readSectionsFromLabel(labels, docNum); while(it.hasNext()) { line = it.next(); if(sections[0].contains(i)) { text = new StringBuilder(); } text.append(line).append("\n"); if(sections[1].contains(i)) { String sectionText = text.toString(); if(sectionText.trim().length() > 0) { addToDocument(sectionText, n++, doc); } text = new StringBuilder(); } i++; } return doc; } catch(IOException ex) { log.error(ex.toString()); throw new RuntimeException(ex.toString(), ex.getCause()); } }  }
MultiSegReader extends RawTextDatasetReader { @Override public Document readDocumentFromFile(Resource file) { try(InputStream in = file.getInputStream()) { CharsetDecoder utf8 = StandardCharsets.UTF_8.newDecoder(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); Iterator<String> it = new LineIterator(br); int i = 0; int n = 0; Document doc = new Document(); doc.setId(file.getFileName()); doc.setSource(file.toString()); doc.setType("multiseg"); String line; StringBuilder text = new StringBuilder(); Pattern filePattern = Pattern.compile("^(.+?/)(.+?)\\.(\\d+)$"); Matcher m = filePattern.matcher(file.toString()); if(!m.matches()) throw new IllegalArgumentException("invalid file name"); String basePath = m.group(1); String baseDoc = m.group(2); int docNum = Integer.parseInt(m.group(3)); Resource labels = file instanceof InternalResource ? Resource.fromJAR(basePath + baseDoc + ".label") : Resource.fromFile(basePath, baseDoc + ".label"); TreeSet[] sections = readSectionsFromLabel(labels, docNum); while(it.hasNext()) { line = it.next(); if(sections[0].contains(i)) { text = new StringBuilder(); } text.append(line).append("\n"); if(sections[1].contains(i)) { String sectionText = text.toString(); if(sectionText.trim().length() > 0) { addToDocument(sectionText, n++, doc); } text = new StringBuilder(); } i++; } return doc; } catch(IOException ex) { log.error(ex.toString()); throw new RuntimeException(ex.toString(), ex.getCause()); } }  @Override Dataset read(Resource path); @Override Document readDocumentFromFile(Resource file); }
MultiSegReader extends RawTextDatasetReader { @Override public Document readDocumentFromFile(Resource file) { try(InputStream in = file.getInputStream()) { CharsetDecoder utf8 = StandardCharsets.UTF_8.newDecoder(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); Iterator<String> it = new LineIterator(br); int i = 0; int n = 0; Document doc = new Document(); doc.setId(file.getFileName()); doc.setSource(file.toString()); doc.setType("multiseg"); String line; StringBuilder text = new StringBuilder(); Pattern filePattern = Pattern.compile("^(.+?/)(.+?)\\.(\\d+)$"); Matcher m = filePattern.matcher(file.toString()); if(!m.matches()) throw new IllegalArgumentException("invalid file name"); String basePath = m.group(1); String baseDoc = m.group(2); int docNum = Integer.parseInt(m.group(3)); Resource labels = file instanceof InternalResource ? Resource.fromJAR(basePath + baseDoc + ".label") : Resource.fromFile(basePath, baseDoc + ".label"); TreeSet[] sections = readSectionsFromLabel(labels, docNum); while(it.hasNext()) { line = it.next(); if(sections[0].contains(i)) { text = new StringBuilder(); } text.append(line).append("\n"); if(sections[1].contains(i)) { String sectionText = text.toString(); if(sectionText.trim().length() > 0) { addToDocument(sectionText, n++, doc); } text = new StringBuilder(); } i++; } return doc; } catch(IOException ex) { log.error(ex.toString()); throw new RuntimeException(ex.toString(), ex.getCause()); } }  @Override Dataset read(Resource path); @Override Document readDocumentFromFile(Resource file);  }