@Test public void testNextDocument() throws Exception { val tempDir = testDir.newFolder(); Resources.copyDirectory("/big/", tempDir); FilenamesLabelAwareIterator iterator = new FilenamesLabelAwareIterator.Builder() .addSourceFolder(tempDir).useAbsolutePathAsLabel(false).build(); List<String> labels = new ArrayList<>(); LabelledDocument doc1 = iterator.nextDocument(); labels.add(doc1.getLabel()); LabelledDocument doc2 = iterator.nextDocument(); labels.add(doc2.getLabel()); LabelledDocument doc3 = iterator.nextDocument(); labels.add(doc3.getLabel()); LabelledDocument doc4 = iterator.nextDocument(); labels.add(doc4.getLabel()); LabelledDocument doc5 = iterator.nextDocument(); labels.add(doc5.getLabel()); LabelledDocument doc6 = iterator.nextDocument(); labels.add(doc6.getLabel()); assertFalse(iterator.hasNextDocument()); System.out.println("Labels: " + labels); assertTrue(labels.contains("coc.txt")); assertTrue(labels.contains("occurrences.txt")); assertTrue(labels.contains("raw_sentences.txt")); assertTrue(labels.contains("tokens.txt")); assertTrue(labels.contains("raw_sentences_2.txt")); assertTrue(labels.contains("rnj.txt")); }
@Override public LabelledDocument nextDocument() { File fileToRead = files.get(position.getAndIncrement()); String label = (absPath) ? fileToRead.getAbsolutePath() : fileToRead.getName(); labelsSource.storeLabel(label); try { LabelledDocument document = new LabelledDocument(); BufferedReader reader = new BufferedReader(new FileReader(fileToRead)); StringBuilder builder = new StringBuilder(); String line = ""; while ((line = reader.readLine()) != null) builder.append(line).append(" "); document.setContent(builder.toString()); document.addLabel(label); try { reader.close(); } catch (Exception e) { throw new RuntimeException(e); } return document; } catch (Exception e) { throw new RuntimeException(e); } }
FilenamesLabelAwareIterator implements LabelAwareIterator { @Override public LabelledDocument nextDocument() { File fileToRead = files.get(position.getAndIncrement()); String label = (absPath) ? fileToRead.getAbsolutePath() : fileToRead.getName(); labelsSource.storeLabel(label); try { LabelledDocument document = new LabelledDocument(); BufferedReader reader = new BufferedReader(new FileReader(fileToRead)); StringBuilder builder = new StringBuilder(); String line = ""; while ((line = reader.readLine()) != null) builder.append(line).append(" "); document.setContent(builder.toString()); document.addLabel(label); try { reader.close(); } catch (Exception e) { throw new RuntimeException(e); } return document; } catch (Exception e) { throw new RuntimeException(e); } } }
FilenamesLabelAwareIterator implements LabelAwareIterator { @Override public LabelledDocument nextDocument() { File fileToRead = files.get(position.getAndIncrement()); String label = (absPath) ? fileToRead.getAbsolutePath() : fileToRead.getName(); labelsSource.storeLabel(label); try { LabelledDocument document = new LabelledDocument(); BufferedReader reader = new BufferedReader(new FileReader(fileToRead)); StringBuilder builder = new StringBuilder(); String line = ""; while ((line = reader.readLine()) != null) builder.append(line).append(" "); document.setContent(builder.toString()); document.addLabel(label); try { reader.close(); } catch (Exception e) { throw new RuntimeException(e); } return document; } catch (Exception e) { throw new RuntimeException(e); } } protected  FilenamesLabelAwareIterator(); protected  FilenamesLabelAwareIterator(@NonNull List<File> files, @NonNull LabelsSource source); }
FilenamesLabelAwareIterator implements LabelAwareIterator { @Override public LabelledDocument nextDocument() { File fileToRead = files.get(position.getAndIncrement()); String label = (absPath) ? fileToRead.getAbsolutePath() : fileToRead.getName(); labelsSource.storeLabel(label); try { LabelledDocument document = new LabelledDocument(); BufferedReader reader = new BufferedReader(new FileReader(fileToRead)); StringBuilder builder = new StringBuilder(); String line = ""; while ((line = reader.readLine()) != null) builder.append(line).append(" "); document.setContent(builder.toString()); document.addLabel(label); try { reader.close(); } catch (Exception e) { throw new RuntimeException(e); } return document; } catch (Exception e) { throw new RuntimeException(e); } } protected  FilenamesLabelAwareIterator(); protected  FilenamesLabelAwareIterator(@NonNull List<File> files, @NonNull LabelsSource source); @Override boolean hasNextDocument(); @Override LabelledDocument nextDocument(); @Override boolean hasNext(); @Override LabelledDocument next(); @Override void remove(); @Override void shutdown(); @Override void reset(); @Override LabelsSource getLabelsSource(); }
FilenamesLabelAwareIterator implements LabelAwareIterator { @Override public LabelledDocument nextDocument() { File fileToRead = files.get(position.getAndIncrement()); String label = (absPath) ? fileToRead.getAbsolutePath() : fileToRead.getName(); labelsSource.storeLabel(label); try { LabelledDocument document = new LabelledDocument(); BufferedReader reader = new BufferedReader(new FileReader(fileToRead)); StringBuilder builder = new StringBuilder(); String line = ""; while ((line = reader.readLine()) != null) builder.append(line).append(" "); document.setContent(builder.toString()); document.addLabel(label); try { reader.close(); } catch (Exception e) { throw new RuntimeException(e); } return document; } catch (Exception e) { throw new RuntimeException(e); } } protected  FilenamesLabelAwareIterator(); protected  FilenamesLabelAwareIterator(@NonNull List<File> files, @NonNull LabelsSource source); @Override boolean hasNextDocument(); @Override LabelledDocument nextDocument(); @Override boolean hasNext(); @Override LabelledDocument next(); @Override void remove(); @Override void shutdown(); @Override void reset(); @Override LabelsSource getLabelsSource();  }