@Test(timeout = 60000L) public void testBagOfWordsVectorizer() throws Exception { val rootDir = testDir.newFolder(); ClassPathResource resource = new ClassPathResource("rootdir/"); resource.copyDirectory(rootDir); LabelAwareSentenceIterator iter = new LabelAwareFileSentenceIterator(rootDir); List<String> labels = Arrays.asList("label1", "label2"); TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); BagOfWordsVectorizer vectorizer = new BagOfWordsVectorizer.Builder().setMinWordFrequency(1) .setStopWords(new ArrayList<String>()).setTokenizerFactory(tokenizerFactory).setIterator(iter) .allowParallelTokenization(false) .build(); vectorizer.fit(); VocabWord word = vectorizer.getVocabCache().wordFor("file."); assumeNotNull(word); assertEquals(word, vectorizer.getVocabCache().tokenFor("file.")); assertEquals(2, vectorizer.getVocabCache().totalNumberOfDocs()); assertEquals(2, word.getSequencesCount()); assertEquals(2, word.getElementFrequency(), 0.1); VocabWord word1 = vectorizer.getVocabCache().wordFor("1"); assertEquals(1, word1.getSequencesCount()); assertEquals(1, word1.getElementFrequency(), 0.1); log.info("Labels used: " + vectorizer.getLabelsSource().getLabels()); assertEquals(2, vectorizer.getLabelsSource().getNumberOfLabelsUsed()); INDArray array = vectorizer.transform("This is 2 file."); log.info("Transformed array: " + array); assertEquals(5, array.columns()); VocabCache<VocabWord> vocabCache = vectorizer.getVocabCache(); assertEquals(2, array.getDouble(vocabCache.tokenFor("This").getIndex()), 0.1); assertEquals(2, array.getDouble(vocabCache.tokenFor("is").getIndex()), 0.1); assertEquals(2, array.getDouble(vocabCache.tokenFor("file.").getIndex()), 0.1); assertEquals(0, array.getDouble(vocabCache.tokenFor("1").getIndex()), 0.1); assertEquals(1, array.getDouble(vocabCache.tokenFor("2").getIndex()), 0.1); DataSet dataSet = vectorizer.vectorize("This is 2 file.", "label2"); assertEquals(array, dataSet.getFeatures()); INDArray labelz = dataSet.getLabels(); log.info("Labels array: " + labelz); int idx2 = Nd4j.getExecutioner().exec(new ArgMax(labelz))[0].getInt(0); dataSet = vectorizer.vectorize("This is 1 file.", "label1"); assertEquals(2, dataSet.getFeatures().getDouble(vocabCache.tokenFor("This").getIndex()), 0.1); assertEquals(2, dataSet.getFeatures().getDouble(vocabCache.tokenFor("is").getIndex()), 0.1); assertEquals(2, dataSet.getFeatures().getDouble(vocabCache.tokenFor("file.").getIndex()), 0.1); assertEquals(1, dataSet.getFeatures().getDouble(vocabCache.tokenFor("1").getIndex()), 0.1); assertEquals(0, dataSet.getFeatures().getDouble(vocabCache.tokenFor("2").getIndex()), 0.1); int idx1 = Nd4j.getExecutioner().exec(new ArgMax(dataSet.getLabels()))[0].getInt(0); assertNotEquals(idx2, idx1); File tempFile = createTempFile("fdsf", "fdfsdf"); tempFile.deleteOnExit(); SerializationUtils.saveObject(vectorizer, tempFile); BagOfWordsVectorizer vectorizer2 = SerializationUtils.readObject(tempFile); vectorizer2.setTokenizerFactory(tokenizerFactory); dataSet = vectorizer2.vectorize("This is 2 file.", "label2"); assertEquals(array, dataSet.getFeatures()); }
protected BagOfWordsVectorizer() { }
BagOfWordsVectorizer extends BaseTextVectorizer { protected BagOfWordsVectorizer() { } }
BagOfWordsVectorizer extends BaseTextVectorizer { protected BagOfWordsVectorizer() { } protected  BagOfWordsVectorizer(); }
BagOfWordsVectorizer extends BaseTextVectorizer { protected BagOfWordsVectorizer() { } protected  BagOfWordsVectorizer(); @Override DataSet vectorize(InputStream is, String label); @Override DataSet vectorize(String text, String label); @Override INDArray transform(String text); @Override INDArray transform(List<String> tokens); @Override DataSet vectorize(File input, String label); @Override DataSet vectorize(); }
BagOfWordsVectorizer extends BaseTextVectorizer { protected BagOfWordsVectorizer() { } protected  BagOfWordsVectorizer(); @Override DataSet vectorize(InputStream is, String label); @Override DataSet vectorize(String text, String label); @Override INDArray transform(String text); @Override INDArray transform(List<String> tokens); @Override DataSet vectorize(File input, String label); @Override DataSet vectorize();  }