@Test public void loadTest1() throws IOException { List<String> lines = new ArrayList<>(); for (int i = 0; i < 10000; i++) { lines.add(String.valueOf(i)); } Path path = TestUtil.tempFileWithData(lines); BlockTextLoader loader = BlockTextLoader.fromPath(path, 1000); int i = 0; List<String> read = new ArrayList<>(); for (TextChunk block : loader) { i++; read.addAll(block.getData()); } Assert.assertEquals(i, 10); Assert.assertEquals(lines, read); loader = BlockTextLoader.fromPath(path, 1001); i = 0; read = new ArrayList<>(); for (TextChunk block : loader) { i++; read.addAll(block.getData()); } Assert.assertEquals(i, 10); Assert.assertEquals(lines, read); loader = BlockTextLoader.fromPath(path, 100000); i = 0; read = new ArrayList<>(); for (TextChunk block : loader) { i++; read.addAll(block.getData()); } Assert.assertEquals(i, 1); Assert.assertEquals(lines, read); Files.delete(path); }
public static BlockTextLoader fromPath(Path corpus, int blockSize) { return new BlockTextLoader(Collections.singletonList(corpus), blockSize); }
BlockTextLoader implements Iterable<TextChunk> { public static BlockTextLoader fromPath(Path corpus, int blockSize) { return new BlockTextLoader(Collections.singletonList(corpus), blockSize); } }
BlockTextLoader implements Iterable<TextChunk> { public static BlockTextLoader fromPath(Path corpus, int blockSize) { return new BlockTextLoader(Collections.singletonList(corpus), blockSize); } BlockTextLoader(List<Path> corpusPaths, int blockSize); }
BlockTextLoader implements Iterable<TextChunk> { public static BlockTextLoader fromPath(Path corpus, int blockSize) { return new BlockTextLoader(Collections.singletonList(corpus), blockSize); } BlockTextLoader(List<Path> corpusPaths, int blockSize); List<Path> getCorpusPaths(); int getBlockSize(); int pathCount(); static BlockTextLoader fromPaths(List<Path> corpora); static BlockTextLoader fromDirectory(Path directoryPath); static BlockTextLoader fromPaths(List<Path> corpora, int blockSize); static BlockTextLoader fromPath(Path corpus, int blockSize); static BlockTextLoader fromDirectoryRoot(
      Path corporaRoot,
      Path folderListFile,
      int blockSize); @Override Iterator<TextChunk> iterator(); static Iterator<TextChunk> singlePathIterator(Path path, int blockSize); static Iterator<TextChunk> iteratorFromCharIndex(
      Path path,
      int blockSize,
      long charIndex); }
BlockTextLoader implements Iterable<TextChunk> { public static BlockTextLoader fromPath(Path corpus, int blockSize) { return new BlockTextLoader(Collections.singletonList(corpus), blockSize); } BlockTextLoader(List<Path> corpusPaths, int blockSize); List<Path> getCorpusPaths(); int getBlockSize(); int pathCount(); static BlockTextLoader fromPaths(List<Path> corpora); static BlockTextLoader fromDirectory(Path directoryPath); static BlockTextLoader fromPaths(List<Path> corpora, int blockSize); static BlockTextLoader fromPath(Path corpus, int blockSize); static BlockTextLoader fromDirectoryRoot(
      Path corporaRoot,
      Path folderListFile,
      int blockSize); @Override Iterator<TextChunk> iterator(); static Iterator<TextChunk> singlePathIterator(Path path, int blockSize); static Iterator<TextChunk> iteratorFromCharIndex(
      Path path,
      int blockSize,
      long charIndex);  }