@Test public void additionalTest(){ String[] corpus = { "This is the first document", "This document is the second document", "And this is the third one", "Is this the first document"}; TfidfVectorizer tfidfVectorizer = new TfidfVectorizer(); Configuration configuration = new Configuration(); configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); configuration.set(MIN_WORD_FREQUENCY,"1"); configuration.set(STOP_WORDS,""); configuration.set(TfidfVectorizer.SMOOTH_IDF, "false"); configuration.set(PREPROCESSOR, LowerCasePreProcessor.class.getName()); tfidfVectorizer.initialize(configuration); List<List<List<Writable>>> input = new ArrayList<>(); List<List<Writable>> seq = new ArrayList<>(); for(String s : corpus){ seq.add(Collections.<Writable>singletonList(new Text(s))); } input.add(seq); CollectionRecordReader crr = new CollectionRecordReader(seq); INDArray arr = tfidfVectorizer.fitTransform(crr); assertArrayEquals(new long[]{4, 9}, arr.shape()); List<String> pyVocab = Arrays.asList("and", "document", "first", "is", "one", "second", "the", "third", "this"); List<Triple<Integer,Integer,Double>> l = new ArrayList<>(); l.add(new Triple<>(0, 8, 1.0)); l.add(new Triple<>(0, 6, 1.0)); l.add(new Triple<>(0, 3, 1.0)); l.add(new Triple<>(0, 2, 1.6931471805599454)); l.add(new Triple<>(0, 1, 1.2876820724517808)); l.add(new Triple<>(1, 8, 1.0)); l.add(new Triple<>(1, 6, 1.0)); l.add(new Triple<>(1, 5, 2.386294361119891)); l.add(new Triple<>(1, 3, 1.0)); l.add(new Triple<>(1, 1, 2.5753641449035616)); l.add(new Triple<>(2, 8, 1.0)); l.add(new Triple<>(2, 7, 2.386294361119891)); l.add(new Triple<>(2, 6, 1.0)); l.add(new Triple<>(2, 4, 2.386294361119891)); l.add(new Triple<>(2, 3, 1.0)); l.add(new Triple<>(2, 0, 2.386294361119891)); l.add(new Triple<>(3, 8, 1.0)); l.add(new Triple<>(3, 6, 1.0)); l.add(new Triple<>(3, 3, 1.0)); l.add(new Triple<>(3, 2, 1.6931471805599454)); l.add(new Triple<>(3, 1, 1.2876820724517808)); INDArray exp = Nd4j.create(DataType.FLOAT, 4, 9); for(Triple<Integer,Integer,Double> t : l){ int wIdx = tfidfVectorizer.getCache().wordIndex(pyVocab.get(t.getSecond())); exp.putScalar(t.getFirst(), wIdx, t.getThird()); } assertEquals(exp, arr); Map<String,Double> idfWeights = new HashMap<>(); idfWeights.put("and", 2.386294361119891); idfWeights.put("document", 1.2876820724517808); idfWeights.put("first", 1.6931471805599454); idfWeights.put("is", 1.0); idfWeights.put("one", 2.386294361119891); idfWeights.put("second", 2.386294361119891); idfWeights.put("the", 1.0); idfWeights.put("third", 2.386294361119891); idfWeights.put("this", 1.0); List<String> vocab = new ArrayList<>(9); for( int i=0; i<9; i++ ){ vocab.add(tfidfVectorizer.getCache().wordAt(i)); } String inputColumnName = "input"; String outputColumnName = "output"; Map<String,Integer> wordIndexMap = new HashMap<>(); for(int i = 0; i < vocab.size(); i++) { wordIndexMap.put(vocab.get(i),i); } TokenizerBagOfWordsTermSequenceIndexTransform transform = new TokenizerBagOfWordsTermSequenceIndexTransform( inputColumnName, outputColumnName, wordIndexMap, idfWeights, false, null, LowerCasePreProcessor.class.getName()); SequenceSchema.Builder sequenceSchemaBuilder = new SequenceSchema.Builder(); sequenceSchemaBuilder.addColumnString("input"); SequenceSchema schema = sequenceSchemaBuilder.build(); assertEquals("input",schema.getName(0)); TransformProcess transformProcess = new TransformProcess.Builder(schema) .transform(transform) .build(); List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); assertEquals(exp.getRow(0, true), arr0); assertEquals(exp.getRow(1, true), arr1); }
@Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); @Override List<Writable> map(List<Writable> writables); @Override Object map(Object input); @Override Object mapSequence(Object sequence); @Override Schema transform(Schema inputSchema); INDArray convert(String text); double tfidfWord(String word, long wordCount, long documentLength); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override Writable map(Writable columnWritable); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); @Override List<Writable> map(List<Writable> writables); @Override Object map(Object input); @Override Object mapSequence(Object sequence); @Override Schema transform(Schema inputSchema); INDArray convert(String text); double tfidfWord(String word, long wordCount, long documentLength); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override Writable map(Writable columnWritable);  }