@Test(timeout = 300000) public void hasNext() throws Exception { SentenceIterator iterator = new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")); SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator).allowMultithreading(true) .tokenizerFactory(factory).build(); Iterator<Sequence<VocabWord>> iter = transformer.iterator(); int cnt = 0; Sequence<VocabWord> sequence = null; while (iter.hasNext()) { sequence = iter.next(); assertNotEquals("Failed on [" + cnt + "] iteration", null, sequence); assertNotEquals("Failed on [" + cnt + "] iteration", 0, sequence.size()); cnt++; } assertEquals(97162, cnt); }
@Override public boolean hasNext() { if (buffer.size() < capacity && iterator.hasNextDocument()) { CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer); Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer); try { buffer.put(futureSequence); } catch (InterruptedException e) { log.error("",e); } } return ( !buffer.isEmpty() || processing.get() > 0); }
ParallelTransformerIterator extends BasicTransformerIterator { @Override public boolean hasNext() { if (buffer.size() < capacity && iterator.hasNextDocument()) { CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer); Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer); try { buffer.put(futureSequence); } catch (InterruptedException e) { log.error("",e); } } return ( !buffer.isEmpty() || processing.get() > 0); } }
ParallelTransformerIterator extends BasicTransformerIterator { @Override public boolean hasNext() { if (buffer.size() < capacity && iterator.hasNextDocument()) { CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer); Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer); try { buffer.put(futureSequence); } catch (InterruptedException e) { log.error("",e); } } return ( !buffer.isEmpty() || processing.get() > 0); } ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer);  ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
                                       boolean allowMultithreading); }
ParallelTransformerIterator extends BasicTransformerIterator { @Override public boolean hasNext() { if (buffer.size() < capacity && iterator.hasNextDocument()) { CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer); Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer); try { buffer.put(futureSequence); } catch (InterruptedException e) { log.error("",e); } } return ( !buffer.isEmpty() || processing.get() > 0); } ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer);  ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
                                       boolean allowMultithreading); @Override void reset(); void shutdown(); @Override boolean hasNext(); @Override Sequence<VocabWord> next(); }
ParallelTransformerIterator extends BasicTransformerIterator { @Override public boolean hasNext() { if (buffer.size() < capacity && iterator.hasNextDocument()) { CallableTransformer transformer = new CallableTransformer(iterator.nextDocument(), sentenceTransformer); Future<Sequence<VocabWord>> futureSequence = executorService.submit(transformer); try { buffer.put(futureSequence); } catch (InterruptedException e) { log.error("",e); } } return ( !buffer.isEmpty() || processing.get() > 0); } ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer);  ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
                                       boolean allowMultithreading); @Override void reset(); void shutdown(); @Override boolean hasNext(); @Override Sequence<VocabWord> next();  }