@Test public void testTransfer_1() { val vocab = new AbstractCache<VocabWord>(); vocab.addToken(new VocabWord(1.0,"alpha")); vocab.addWordToIndex(0, "alpha"); vocab.addToken(new VocabWord(2.0,"beta")); vocab.addWordToIndex(5, "beta"); vocab.addToken(new VocabWord(3.0,"gamma")); vocab.addWordToIndex(10, "gamma"); val constructor = new VocabConstructor.Builder<VocabWord>() .build(); val result = constructor.transferVocabulary(vocab, true); assertEquals(3, result.numWords()); assertEquals("gamma", result.wordAtIndex(0)); assertEquals("beta", result.wordAtIndex(1)); assertEquals("alpha", result.wordAtIndex(2)); }
public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) { val result = cache != null ? cache : new AbstractCache.Builder<T>().build(); for (val v: vocabCache.tokens()) { result.addToken(v); if (v.getIndex() >= 0) result.addWordToIndex(v.getIndex(), v.getLabel()); else result.addWordToIndex(result.numWords(), v.getLabel()); } if (buildHuffman) { val huffman = new Huffman(result.vocabWords()); huffman.build(); huffman.applyIndexes(result); } return result; }
VocabConstructor { public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) { val result = cache != null ? cache : new AbstractCache.Builder<T>().build(); for (val v: vocabCache.tokens()) { result.addToken(v); if (v.getIndex() >= 0) result.addWordToIndex(v.getIndex(), v.getLabel()); else result.addWordToIndex(result.numWords(), v.getLabel()); } if (buildHuffman) { val huffman = new Huffman(result.vocabWords()); huffman.build(); huffman.applyIndexes(result); } return result; } }
VocabConstructor { public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) { val result = cache != null ? cache : new AbstractCache.Builder<T>().build(); for (val v: vocabCache.tokens()) { result.addToken(v); if (v.getIndex() >= 0) result.addWordToIndex(v.getIndex(), v.getLabel()); else result.addWordToIndex(result.numWords(), v.getLabel()); } if (buildHuffman) { val huffman = new Huffman(result.vocabWords()); huffman.build(); huffman.applyIndexes(result); } return result; } private  VocabConstructor(); }
VocabConstructor { public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) { val result = cache != null ? cache : new AbstractCache.Builder<T>().build(); for (val v: vocabCache.tokens()) { result.addToken(v); if (v.getIndex() >= 0) result.addWordToIndex(v.getIndex(), v.getLabel()); else result.addWordToIndex(result.numWords(), v.getLabel()); } if (buildHuffman) { val huffman = new Huffman(result.vocabWords()); huffman.build(); huffman.applyIndexes(result); } return result; } private  VocabConstructor(); @SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels); long getNumberOfSequences(); VocabCache<T> buildMergedVocabulary(@NonNull VocabCache<T> vocabCache, boolean fetchLabels); VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman); void processDocument(AbstractCache<T> targetVocab, Sequence<T> document,
                                AtomicLong finalCounter, AtomicLong loopCounter); VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree); }
VocabConstructor { public VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman) { val result = cache != null ? cache : new AbstractCache.Builder<T>().build(); for (val v: vocabCache.tokens()) { result.addToken(v); if (v.getIndex() >= 0) result.addWordToIndex(v.getIndex(), v.getLabel()); else result.addWordToIndex(result.numWords(), v.getLabel()); } if (buildHuffman) { val huffman = new Huffman(result.vocabWords()); huffman.build(); huffman.applyIndexes(result); } return result; } private  VocabConstructor(); @SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels); long getNumberOfSequences(); VocabCache<T> buildMergedVocabulary(@NonNull VocabCache<T> vocabCache, boolean fetchLabels); VocabCache<T> transferVocabulary(@NonNull VocabCache<T> vocabCache, boolean buildHuffman); void processDocument(AbstractCache<T> targetVocab, Sequence<T> document,
                                AtomicLong finalCounter, AtomicLong loopCounter); VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree);  }