@Ignore @Test public void testGoogleModelForInference() throws Exception { WordVectors googleVectors = WordVectorSerializer.readWord2VecModel(new File("/ext/GoogleNews-vectors-negative300.bin.gz")); TokenizerFactory t = new DefaultTokenizerFactory(); t.setTokenPreProcessor(new CommonPreprocessor()); ParagraphVectors pv = new ParagraphVectors.Builder().tokenizerFactory(t).iterations(10).useHierarchicSoftmax(false) .trainWordVectors(false).iterations(10).useExistingWordVectors(googleVectors) .negativeSample(10).sequenceLearningAlgorithm(new DM<VocabWord>()).build(); INDArray vec1 = pv.inferVector("This text is pretty awesome"); INDArray vec2 = pv.inferVector("Fantastic process of crazy things happening inside just for history purposes"); log.info("vec1/vec2: {}", Transforms.cosineSim(vec1, vec2)); }
public INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations) { if (tokenizerFactory == null) throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call"); if (this.vocab == null || this.vocab.numWords() == 0) reassignExistingModel(); List<String> tokens = tokenizerFactory.create(text).getTokens(); List<VocabWord> document = new ArrayList<>(); for (String token : tokens) { if (vocab.containsWord(token)) { document.add(vocab.wordFor(token)); } } if (document.isEmpty()) throw new ND4JIllegalStateException("Text passed for inference has no matches in model vocabulary."); return inferVector(document, learningRate, minLearningRate, iterations); }
ParagraphVectors extends Word2Vec { public INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations) { if (tokenizerFactory == null) throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call"); if (this.vocab == null || this.vocab.numWords() == 0) reassignExistingModel(); List<String> tokens = tokenizerFactory.create(text).getTokens(); List<VocabWord> document = new ArrayList<>(); for (String token : tokens) { if (vocab.containsWord(token)) { document.add(vocab.wordFor(token)); } } if (document.isEmpty()) throw new ND4JIllegalStateException("Text passed for inference has no matches in model vocabulary."); return inferVector(document, learningRate, minLearningRate, iterations); } }
ParagraphVectors extends Word2Vec { public INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations) { if (tokenizerFactory == null) throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call"); if (this.vocab == null || this.vocab.numWords() == 0) reassignExistingModel(); List<String> tokens = tokenizerFactory.create(text).getTokens(); List<VocabWord> document = new ArrayList<>(); for (String token : tokens) { if (vocab.containsWord(token)) { document.add(vocab.wordFor(token)); } } if (document.isEmpty()) throw new ND4JIllegalStateException("Text passed for inference has no matches in model vocabulary."); return inferVector(document, learningRate, minLearningRate, iterations); } protected  ParagraphVectors(); }
ParagraphVectors extends Word2Vec { public INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations) { if (tokenizerFactory == null) throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call"); if (this.vocab == null || this.vocab.numWords() == 0) reassignExistingModel(); List<String> tokens = tokenizerFactory.create(text).getTokens(); List<VocabWord> document = new ArrayList<>(); for (String token : tokens) { if (vocab.containsWord(token)) { document.add(vocab.wordFor(token)); } } if (document.isEmpty()) throw new ND4JIllegalStateException("Text passed for inference has no matches in model vocabulary."); return inferVector(document, learningRate, minLearningRate, iterations); } protected  ParagraphVectors(); @Deprecated String predict(String rawText); void setSequenceIterator(@NonNull SequenceIterator<VocabWord> iterator); String predict(LabelledDocument document); void extractLabels(); INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations); INDArray inferVector(LabelledDocument document, double learningRate, double minLearningRate,
                    int iterations); INDArray inferVector(@NonNull List<VocabWord> document, double learningRate, double minLearningRate,
                    int iterations); INDArray inferVector(String text); INDArray inferVector(LabelledDocument document); INDArray inferVector(@NonNull List<VocabWord> document); Future<Pair<String, INDArray>> inferVectorBatched(@NonNull LabelledDocument document); Future<INDArray> inferVectorBatched(@NonNull String document); List<INDArray> inferVectorBatched(@NonNull List<String> documents); String predict(List<VocabWord> document); Collection<String> predictSeveral(@NonNull LabelledDocument document, int limit); Collection<String> predictSeveral(String rawText, int limit); Collection<String> predictSeveral(List<VocabWord> document, int limit); Collection<String> nearestLabels(LabelledDocument document, int topN); Collection<String> nearestLabels(@NonNull String rawText, int topN); Collection<String> nearestLabels(@NonNull Collection<VocabWord> document, int topN); Collection<String> nearestLabels(INDArray labelVector, int topN); @Deprecated double similarityToLabel(String rawText, String label); @Override void fit(); double similarityToLabel(LabelledDocument document, String label); double similarityToLabel(List<VocabWord> document, String label); String toJson(); static ParagraphVectors fromJson(String jsonString); }
ParagraphVectors extends Word2Vec { public INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations) { if (tokenizerFactory == null) throw new IllegalStateException("TokenizerFactory should be defined, prior to predict() call"); if (this.vocab == null || this.vocab.numWords() == 0) reassignExistingModel(); List<String> tokens = tokenizerFactory.create(text).getTokens(); List<VocabWord> document = new ArrayList<>(); for (String token : tokens) { if (vocab.containsWord(token)) { document.add(vocab.wordFor(token)); } } if (document.isEmpty()) throw new ND4JIllegalStateException("Text passed for inference has no matches in model vocabulary."); return inferVector(document, learningRate, minLearningRate, iterations); } protected  ParagraphVectors(); @Deprecated String predict(String rawText); void setSequenceIterator(@NonNull SequenceIterator<VocabWord> iterator); String predict(LabelledDocument document); void extractLabels(); INDArray inferVector(String text, double learningRate, double minLearningRate, int iterations); INDArray inferVector(LabelledDocument document, double learningRate, double minLearningRate,
                    int iterations); INDArray inferVector(@NonNull List<VocabWord> document, double learningRate, double minLearningRate,
                    int iterations); INDArray inferVector(String text); INDArray inferVector(LabelledDocument document); INDArray inferVector(@NonNull List<VocabWord> document); Future<Pair<String, INDArray>> inferVectorBatched(@NonNull LabelledDocument document); Future<INDArray> inferVectorBatched(@NonNull String document); List<INDArray> inferVectorBatched(@NonNull List<String> documents); String predict(List<VocabWord> document); Collection<String> predictSeveral(@NonNull LabelledDocument document, int limit); Collection<String> predictSeveral(String rawText, int limit); Collection<String> predictSeveral(List<VocabWord> document, int limit); Collection<String> nearestLabels(LabelledDocument document, int topN); Collection<String> nearestLabels(@NonNull String rawText, int topN); Collection<String> nearestLabels(@NonNull Collection<VocabWord> document, int topN); Collection<String> nearestLabels(INDArray labelVector, int topN); @Deprecated double similarityToLabel(String rawText, String label); @Override void fit(); double similarityToLabel(LabelledDocument document, String label); double similarityToLabel(List<VocabWord> document, String label); String toJson(); static ParagraphVectors fromJson(String jsonString);  }