@Test public void testSequenceExecution() { String[] corpus = { "This is very strange".toLowerCase(), "This is very nice".toLowerCase() }; List<List<List<Writable>>> input = new ArrayList<>(); input.add(Arrays.asList(Arrays.<Writable>asList(new Text(corpus[0])),Arrays.<Writable>asList(new Text(corpus[1])))); Map<String,Double> idfMapNoSmooth = new HashMap<>(); idfMapNoSmooth.put("is",1.0); idfMapNoSmooth.put("nice",1.6931471805599454); idfMapNoSmooth.put("strange",1.6931471805599454); idfMapNoSmooth.put("this",1.0); idfMapNoSmooth.put("very",1.0); Map<String,Double> idfMapSmooth = new HashMap<>(); idfMapSmooth.put("is",1.0); idfMapSmooth.put("nice",1.4054651081081644); idfMapSmooth.put("strange",1.4054651081081644); idfMapSmooth.put("this",1.0); idfMapSmooth.put("very",1.0); TfidfVectorizer tfidfVectorizer = new TfidfVectorizer(); Configuration configuration = new Configuration(); configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); configuration.set(MIN_WORD_FREQUENCY,"1"); configuration.set(STOP_WORDS,""); configuration.set(TfidfVectorizer.SMOOTH_IDF, "false"); tfidfVectorizer.initialize(configuration); CollectionRecordReader collectionRecordReader = new CollectionRecordReader(input.get(0)); INDArray array = tfidfVectorizer.fitTransform(collectionRecordReader); INDArray expNoSmooth = Nd4j.create(DataType.FLOAT, 2, 5); VocabCache vc = tfidfVectorizer.getCache(); expNoSmooth.putScalar(0, vc.wordIndex("very"), 1.0); expNoSmooth.putScalar(0, vc.wordIndex("this"), 1.0); expNoSmooth.putScalar(0, vc.wordIndex("strange"), 1.6931471805599454); expNoSmooth.putScalar(0, vc.wordIndex("is"), 1.0); expNoSmooth.putScalar(1, vc.wordIndex("very"), 1.0); expNoSmooth.putScalar(1, vc.wordIndex("this"), 1.0); expNoSmooth.putScalar(1, vc.wordIndex("nice"), 1.6931471805599454); expNoSmooth.putScalar(1, vc.wordIndex("is"), 1.0); assertEquals(expNoSmooth, array); tfidfVectorizer = new TfidfVectorizer(); configuration = new Configuration(); configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); configuration.set(MIN_WORD_FREQUENCY,"1"); configuration.set(STOP_WORDS,""); configuration.set(TfidfVectorizer.SMOOTH_IDF, "true"); tfidfVectorizer.initialize(configuration); collectionRecordReader.reset(); array = tfidfVectorizer.fitTransform(collectionRecordReader); INDArray expSmooth = Nd4j.create(DataType.FLOAT, 2, 5); expSmooth.putScalar(0, vc.wordIndex("very"), 1.0); expSmooth.putScalar(0, vc.wordIndex("this"), 1.0); expSmooth.putScalar(0, vc.wordIndex("strange"), 1.4054651081081644); expSmooth.putScalar(0, vc.wordIndex("is"), 1.0); expSmooth.putScalar(1, vc.wordIndex("very"), 1.0); expSmooth.putScalar(1, vc.wordIndex("this"), 1.0); expSmooth.putScalar(1, vc.wordIndex("nice"), 1.4054651081081644); expSmooth.putScalar(1, vc.wordIndex("is"), 1.0); assertEquals(expSmooth, array); List<String> vocab = new ArrayList<>(5); for( int i=0; i<5; i++ ){ vocab.add(vc.wordAt(i)); } String inputColumnName = "input"; String outputColumnName = "output"; Map<String,Integer> wordIndexMap = new HashMap<>(); for(int i = 0; i < vocab.size(); i++) { wordIndexMap.put(vocab.get(i),i); } TokenizerBagOfWordsTermSequenceIndexTransform tokenizerBagOfWordsTermSequenceIndexTransform = new TokenizerBagOfWordsTermSequenceIndexTransform( inputColumnName, outputColumnName, wordIndexMap, idfMapNoSmooth, false, null, null); SequenceSchema.Builder sequenceSchemaBuilder = new SequenceSchema.Builder(); sequenceSchemaBuilder.addColumnString("input"); SequenceSchema schema = sequenceSchemaBuilder.build(); assertEquals("input",schema.getName(0)); TransformProcess transformProcess = new TransformProcess.Builder(schema) .transform(tokenizerBagOfWordsTermSequenceIndexTransform) .build(); List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); assertEquals(expNoSmooth.getRow(0, true), arr0); assertEquals(expNoSmooth.getRow(1, true), arr1); tokenizerBagOfWordsTermSequenceIndexTransform = new TokenizerBagOfWordsTermSequenceIndexTransform( inputColumnName, outputColumnName, wordIndexMap, idfMapSmooth, false, null, null); schema = (SequenceSchema) new SequenceSchema.Builder().addColumnString("input").build(); transformProcess = new TransformProcess.Builder(schema) .transform(tokenizerBagOfWordsTermSequenceIndexTransform) .build(); execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); assertEquals(expSmooth.getRow(0, true), arr0); assertEquals(expSmooth.getRow(1, true), arr1); String json = transformProcess.toJson(); TransformProcess fromJson = TransformProcess.fromJson(json); assertEquals(transformProcess, fromJson); List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, fromJson); INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get(); INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get(); assertEquals(expSmooth.getRow(0, true), arr0a); assertEquals(expSmooth.getRow(1, true), arr1a); }
@Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); @Override List<Writable> map(List<Writable> writables); @Override Object map(Object input); @Override Object mapSequence(Object sequence); @Override Schema transform(Schema inputSchema); INDArray convert(String text); double tfidfWord(String word, long wordCount, long documentLength); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override Writable map(Writable columnWritable); }
TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { @Override public Schema transform(Schema inputSchema) { Schema.Builder newSchema = new Schema.Builder(); for(int i = 0; i < inputSchema.numColumns(); i++) { if(inputSchema.getName(i).equals(this.columnName)) { newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); } else { newSchema.addColumn(inputSchema.getMetaData(i)); } } return newSchema.build(); } @JsonCreator  TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName,
                                                         @JsonProperty("newColumnName") String newColumnName,
                                                         @JsonProperty("wordIndexMap") Map<String,Integer> wordIndexMap,
                                                         @JsonProperty("idfMap") Map<String,Double> idfMap,
                                                         @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown,
                                                         @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass,
                                                         @JsonProperty("preprocessorClass") String preprocessorClass); @Override List<Writable> map(List<Writable> writables); @Override Object map(Object input); @Override Object mapSequence(Object sequence); @Override Schema transform(Schema inputSchema); INDArray convert(String text); double tfidfWord(String word, long wordCount, long documentLength); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override Writable map(Writable columnWritable);  }