@Test public void testBuildJointVocabulary1() throws Exception { File inputFile = Resources.asFile("big/raw_sentences.txt"); SentenceIterator iter = new BasicLineIterator(inputFile); VocabCache<VocabWord> cache = new AbstractCache.Builder<VocabWord>().build(); SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iter).tokenizerFactory(t).build(); AbstractSequenceIterator<VocabWord> sequenceIterator = new AbstractSequenceIterator.Builder<>(transformer).build(); VocabConstructor<VocabWord> constructor = new VocabConstructor.Builder<VocabWord>() .addSource(sequenceIterator, 0).useAdaGrad(false).setTargetVocabCache(cache).build(); constructor.buildJointVocabulary(true, false); assertEquals(244, cache.numWords()); assertEquals(0, cache.totalWordOccurrences()); }
public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) { long lastTime = System.currentTimeMillis(); long lastSequences = 0; long lastElements = 0; long startTime = lastTime; AtomicLong parsedCount = new AtomicLong(0); if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!"); if (cache == null) cache = new AbstractCache.Builder<T>().build(); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); final AtomicLong loopCounter = new AtomicLong(0); AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build(); int cnt = 0; int numProc = Runtime.getRuntime().availableProcessors(); int numThreads = Math.max(numProc / 2, 2); PriorityScheduler executorService = new PriorityScheduler(numThreads); final AtomicLong execCounter = new AtomicLong(0); final AtomicLong finCounter = new AtomicLong(0); for (VocabSource<T> source : sources) { SequenceIterator<T> iterator = source.getIterator(); iterator.reset(); log.debug("Trying source iterator: [" + cnt + "]"); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); cnt++; AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build(); int sequences = 0; while (iterator.hasMoreSequences()) { Sequence<T> document = iterator.nextSequence(); seqCount.incrementAndGet(); parsedCount.addAndGet(document.size()); tempHolder.incrementTotalDocCount(); execCounter.incrementAndGet(); if (allowParallelBuilder) { executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter)); while (execCounter.get() - finCounter.get() > numProc) { ThreadUtils.uncheckedSleep(1); } } else { processDocument(tempHolder, document, finCounter, loopCounter); } sequences++; if (seqCount.get() % 100000 == 0) { long currentTime = System.currentTimeMillis(); long currentSequences = seqCount.get(); long currentElements = parsedCount.get(); double seconds = (currentTime - lastTime) / (double) 1000; double seqPerSec = (currentSequences - lastSequences) / seconds; double elPerSec = (currentElements - lastElements) / seconds; log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};", seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec), String.format("%.2f", elPerSec)); lastTime = currentTime; lastElements = currentElements; lastSequences = currentSequences; } if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) { log.info("Starting scavenger..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2)); loopCounter.set(0); } } log.debug("Waiting till all processes stop..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); if (source.getMinWordFrequency() > 0) { filterVocab(tempHolder, source.getMinWordFrequency()); } log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); topHolder.importVocabulary(tempHolder); } System.gc(); cache.importVocabulary(topHolder); if (unk != null) { log.info("Adding UNK element to vocab..."); unk.setSpecial(true); cache.addToken(unk); } if (resetCounters) { for (T element : cache.vocabWords()) { element.setElementFrequency(0); } cache.updateWordsOccurrences(); } if (buildHuffmanTree) { if (limit > 0) { val words = new ArrayList<T>(cache.vocabWords()); Collections.sort(words); for (val element : words) { if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel()) cache.removeElement(element.getLabel()); } } val huffman = new Huffman(cache.vocabWords()); huffman.build(); huffman.applyIndexes(cache); } executorService.shutdown(); System.gc(); long endSequences = seqCount.get(); long endTime = System.currentTimeMillis(); double seconds = (endTime - startTime) / (double) 1000; double seqPerSec = endSequences / seconds; log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(), cache.numWords(), String.format("%.2f", seqPerSec)); return cache; }
VocabConstructor { public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) { long lastTime = System.currentTimeMillis(); long lastSequences = 0; long lastElements = 0; long startTime = lastTime; AtomicLong parsedCount = new AtomicLong(0); if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!"); if (cache == null) cache = new AbstractCache.Builder<T>().build(); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); final AtomicLong loopCounter = new AtomicLong(0); AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build(); int cnt = 0; int numProc = Runtime.getRuntime().availableProcessors(); int numThreads = Math.max(numProc / 2, 2); PriorityScheduler executorService = new PriorityScheduler(numThreads); final AtomicLong execCounter = new AtomicLong(0); final AtomicLong finCounter = new AtomicLong(0); for (VocabSource<T> source : sources) { SequenceIterator<T> iterator = source.getIterator(); iterator.reset(); log.debug("Trying source iterator: [" + cnt + "]"); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); cnt++; AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build(); int sequences = 0; while (iterator.hasMoreSequences()) { Sequence<T> document = iterator.nextSequence(); seqCount.incrementAndGet(); parsedCount.addAndGet(document.size()); tempHolder.incrementTotalDocCount(); execCounter.incrementAndGet(); if (allowParallelBuilder) { executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter)); while (execCounter.get() - finCounter.get() > numProc) { ThreadUtils.uncheckedSleep(1); } } else { processDocument(tempHolder, document, finCounter, loopCounter); } sequences++; if (seqCount.get() % 100000 == 0) { long currentTime = System.currentTimeMillis(); long currentSequences = seqCount.get(); long currentElements = parsedCount.get(); double seconds = (currentTime - lastTime) / (double) 1000; double seqPerSec = (currentSequences - lastSequences) / seconds; double elPerSec = (currentElements - lastElements) / seconds; log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};", seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec), String.format("%.2f", elPerSec)); lastTime = currentTime; lastElements = currentElements; lastSequences = currentSequences; } if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) { log.info("Starting scavenger..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2)); loopCounter.set(0); } } log.debug("Waiting till all processes stop..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); if (source.getMinWordFrequency() > 0) { filterVocab(tempHolder, source.getMinWordFrequency()); } log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); topHolder.importVocabulary(tempHolder); } System.gc(); cache.importVocabulary(topHolder); if (unk != null) { log.info("Adding UNK element to vocab..."); unk.setSpecial(true); cache.addToken(unk); } if (resetCounters) { for (T element : cache.vocabWords()) { element.setElementFrequency(0); } cache.updateWordsOccurrences(); } if (buildHuffmanTree) { if (limit > 0) { val words = new ArrayList<T>(cache.vocabWords()); Collections.sort(words); for (val element : words) { if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel()) cache.removeElement(element.getLabel()); } } val huffman = new Huffman(cache.vocabWords()); huffman.build(); huffman.applyIndexes(cache); } executorService.shutdown(); System.gc(); long endSequences = seqCount.get(); long endTime = System.currentTimeMillis(); double seconds = (endTime - startTime) / (double) 1000; double seqPerSec = endSequences / seconds; log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(), cache.numWords(), String.format("%.2f", seqPerSec)); return cache; } }
VocabConstructor { public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) { long lastTime = System.currentTimeMillis(); long lastSequences = 0; long lastElements = 0; long startTime = lastTime; AtomicLong parsedCount = new AtomicLong(0); if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!"); if (cache == null) cache = new AbstractCache.Builder<T>().build(); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); final AtomicLong loopCounter = new AtomicLong(0); AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build(); int cnt = 0; int numProc = Runtime.getRuntime().availableProcessors(); int numThreads = Math.max(numProc / 2, 2); PriorityScheduler executorService = new PriorityScheduler(numThreads); final AtomicLong execCounter = new AtomicLong(0); final AtomicLong finCounter = new AtomicLong(0); for (VocabSource<T> source : sources) { SequenceIterator<T> iterator = source.getIterator(); iterator.reset(); log.debug("Trying source iterator: [" + cnt + "]"); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); cnt++; AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build(); int sequences = 0; while (iterator.hasMoreSequences()) { Sequence<T> document = iterator.nextSequence(); seqCount.incrementAndGet(); parsedCount.addAndGet(document.size()); tempHolder.incrementTotalDocCount(); execCounter.incrementAndGet(); if (allowParallelBuilder) { executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter)); while (execCounter.get() - finCounter.get() > numProc) { ThreadUtils.uncheckedSleep(1); } } else { processDocument(tempHolder, document, finCounter, loopCounter); } sequences++; if (seqCount.get() % 100000 == 0) { long currentTime = System.currentTimeMillis(); long currentSequences = seqCount.get(); long currentElements = parsedCount.get(); double seconds = (currentTime - lastTime) / (double) 1000; double seqPerSec = (currentSequences - lastSequences) / seconds; double elPerSec = (currentElements - lastElements) / seconds; log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};", seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec), String.format("%.2f", elPerSec)); lastTime = currentTime; lastElements = currentElements; lastSequences = currentSequences; } if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) { log.info("Starting scavenger..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2)); loopCounter.set(0); } } log.debug("Waiting till all processes stop..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); if (source.getMinWordFrequency() > 0) { filterVocab(tempHolder, source.getMinWordFrequency()); } log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); topHolder.importVocabulary(tempHolder); } System.gc(); cache.importVocabulary(topHolder); if (unk != null) { log.info("Adding UNK element to vocab..."); unk.setSpecial(true); cache.addToken(unk); } if (resetCounters) { for (T element : cache.vocabWords()) { element.setElementFrequency(0); } cache.updateWordsOccurrences(); } if (buildHuffmanTree) { if (limit > 0) { val words = new ArrayList<T>(cache.vocabWords()); Collections.sort(words); for (val element : words) { if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel()) cache.removeElement(element.getLabel()); } } val huffman = new Huffman(cache.vocabWords()); huffman.build(); huffman.applyIndexes(cache); } executorService.shutdown(); System.gc(); long endSequences = seqCount.get(); long endTime = System.currentTimeMillis(); double seconds = (endTime - startTime) / (double) 1000; double seqPerSec = endSequences / seconds; log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(), cache.numWords(), String.format("%.2f", seqPerSec)); return cache; } private  VocabConstructor(); }
VocabConstructor { public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) { long lastTime = System.currentTimeMillis(); long lastSequences = 0; long lastElements = 0; long startTime = lastTime; AtomicLong parsedCount = new AtomicLong(0); if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!"); if (cache == null) cache = new AbstractCache.Builder<T>().build(); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); final AtomicLong loopCounter = new AtomicLong(0); AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build(); int cnt = 0; int numProc = Runtime.getRuntime().availableProcessors(); int numThreads = Math.max(numProc / 2, 2); PriorityScheduler executorService = new PriorityScheduler(numThreads); final AtomicLong execCounter = new AtomicLong(0); final AtomicLong finCounter = new AtomicLong(0); for (VocabSource<T> source : sources) { SequenceIterator<T> iterator = source.getIterator(); iterator.reset(); log.debug("Trying source iterator: [" + cnt + "]"); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); cnt++; AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build(); int sequences = 0; while (iterator.hasMoreSequences()) { Sequence<T> document = iterator.nextSequence(); seqCount.incrementAndGet(); parsedCount.addAndGet(document.size()); tempHolder.incrementTotalDocCount(); execCounter.incrementAndGet(); if (allowParallelBuilder) { executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter)); while (execCounter.get() - finCounter.get() > numProc) { ThreadUtils.uncheckedSleep(1); } } else { processDocument(tempHolder, document, finCounter, loopCounter); } sequences++; if (seqCount.get() % 100000 == 0) { long currentTime = System.currentTimeMillis(); long currentSequences = seqCount.get(); long currentElements = parsedCount.get(); double seconds = (currentTime - lastTime) / (double) 1000; double seqPerSec = (currentSequences - lastSequences) / seconds; double elPerSec = (currentElements - lastElements) / seconds; log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};", seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec), String.format("%.2f", elPerSec)); lastTime = currentTime; lastElements = currentElements; lastSequences = currentSequences; } if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) { log.info("Starting scavenger..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2)); loopCounter.set(0); } } log.debug("Waiting till all processes stop..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); if (source.getMinWordFrequency() > 0) { filterVocab(tempHolder, source.getMinWordFrequency()); } log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); topHolder.importVocabulary(tempHolder); } System.gc(); cache.importVocabulary(topHolder); if (unk != null) { log.info("Adding UNK element to vocab..."); unk.setSpecial(true); cache.addToken(unk); } if (resetCounters) { for (T element : cache.vocabWords()) { element.setElementFrequency(0); } cache.updateWordsOccurrences(); } if (buildHuffmanTree) { if (limit > 0) { val words = new ArrayList<T>(cache.vocabWords()); Collections.sort(words); for (val element : words) { if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel()) cache.removeElement(element.getLabel()); } } val huffman = new Huffman(cache.vocabWords()); huffman.build(); huffman.applyIndexes(cache); } executorService.shutdown(); System.gc(); long endSequences = seqCount.get(); long endTime = System.currentTimeMillis(); double seconds = (endTime - startTime) / (double) 1000; double seqPerSec = endSequences / seconds; log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(), cache.numWords(), String.format("%.2f", seqPerSec)); return cache; } private  VocabConstructor(); @SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels); long getNumberOfSequences(); VocabCache<T> buildMergedVocabulary(@NonNull VocabCache<T> vocabCache, boolean fetchLabels); VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman); void processDocument(AbstractCache<T> targetVocab, Sequence<T> document,
                                AtomicLong finalCounter, AtomicLong loopCounter); VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree); }
VocabConstructor { public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) { long lastTime = System.currentTimeMillis(); long lastSequences = 0; long lastElements = 0; long startTime = lastTime; AtomicLong parsedCount = new AtomicLong(0); if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!"); if (cache == null) cache = new AbstractCache.Builder<T>().build(); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); final AtomicLong loopCounter = new AtomicLong(0); AbstractCache<T> topHolder = new AbstractCache.Builder<T>().minElementFrequency(0).build(); int cnt = 0; int numProc = Runtime.getRuntime().availableProcessors(); int numThreads = Math.max(numProc / 2, 2); PriorityScheduler executorService = new PriorityScheduler(numThreads); final AtomicLong execCounter = new AtomicLong(0); final AtomicLong finCounter = new AtomicLong(0); for (VocabSource<T> source : sources) { SequenceIterator<T> iterator = source.getIterator(); iterator.reset(); log.debug("Trying source iterator: [" + cnt + "]"); log.debug("Target vocab size before building: [" + cache.numWords() + "]"); cnt++; AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build(); int sequences = 0; while (iterator.hasMoreSequences()) { Sequence<T> document = iterator.nextSequence(); seqCount.incrementAndGet(); parsedCount.addAndGet(document.size()); tempHolder.incrementTotalDocCount(); execCounter.incrementAndGet(); if (allowParallelBuilder) { executorService.execute(new VocabRunnable(tempHolder, document, finCounter, loopCounter)); while (execCounter.get() - finCounter.get() > numProc) { ThreadUtils.uncheckedSleep(1); } } else { processDocument(tempHolder, document, finCounter, loopCounter); } sequences++; if (seqCount.get() % 100000 == 0) { long currentTime = System.currentTimeMillis(); long currentSequences = seqCount.get(); long currentElements = parsedCount.get(); double seconds = (currentTime - lastTime) / (double) 1000; double seqPerSec = (currentSequences - lastSequences) / seconds; double elPerSec = (currentElements - lastElements) / seconds; log.info("Sequences checked: [{}]; Current vocabulary size: [{}]; Sequences/sec: {}; Words/sec: {};", seqCount.get(), tempHolder.numWords(), String.format("%.2f", seqPerSec), String.format("%.2f", elPerSec)); lastTime = currentTime; lastElements = currentElements; lastSequences = currentSequences; } if (enableScavenger && loopCounter.get() >= 2000000 && tempHolder.numWords() > 10000000) { log.info("Starting scavenger..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } filterVocab(tempHolder, Math.max(1, source.getMinWordFrequency() / 2)); loopCounter.set(0); } } log.debug("Waiting till all processes stop..."); while (execCounter.get() != finCounter.get()) { ThreadUtils.uncheckedSleep(1); } log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); if (source.getMinWordFrequency() > 0) { filterVocab(tempHolder, source.getMinWordFrequency()); } log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "], NumWords: [" + tempHolder.totalWordOccurrences() + "], sequences parsed: [" + seqCount.get() + "], counter: [" + parsedCount.get() + "]"); topHolder.importVocabulary(tempHolder); } System.gc(); cache.importVocabulary(topHolder); if (unk != null) { log.info("Adding UNK element to vocab..."); unk.setSpecial(true); cache.addToken(unk); } if (resetCounters) { for (T element : cache.vocabWords()) { element.setElementFrequency(0); } cache.updateWordsOccurrences(); } if (buildHuffmanTree) { if (limit > 0) { val words = new ArrayList<T>(cache.vocabWords()); Collections.sort(words); for (val element : words) { if (element.getIndex() > limit && !element.isSpecial() && !element.isLabel()) cache.removeElement(element.getLabel()); } } val huffman = new Huffman(cache.vocabWords()); huffman.build(); huffman.applyIndexes(cache); } executorService.shutdown(); System.gc(); long endSequences = seqCount.get(); long endTime = System.currentTimeMillis(); double seconds = (endTime - startTime) / (double) 1000; double seqPerSec = endSequences / seconds; log.info("Sequences checked: [{}], Current vocabulary size: [{}]; Sequences/sec: [{}];", seqCount.get(), cache.numWords(), String.format("%.2f", seqPerSec)); return cache; } private  VocabConstructor(); @SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels); long getNumberOfSequences(); VocabCache<T> buildMergedVocabulary(@NonNull VocabCache<T> vocabCache, boolean fetchLabels); VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman); void processDocument(AbstractCache<T> targetVocab, Sequence<T> document,
                                AtomicLong finalCounter, AtomicLong loopCounter); VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree);  }