@Test public void testGazeteerTransform(){ String[] corpus = { "hello I like apple".toLowerCase(), "cherry date eggplant potato".toLowerCase() }; List<String> words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant"); GazeteerTransform t = new GazeteerTransform("words", "out", words); SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder() .addColumnString("words").build(); TransformProcess tp = new TransformProcess.Builder(schema) .transform(t) .build(); List<List<List<Writable>>> input = new ArrayList<>(); for(String s : corpus){ String[] split = s.split(" "); List<List<Writable>> seq = new ArrayList<>(); for(String s2 : split){ seq.add(Collections.<Writable>singletonList(new Text(s2))); } input.add(seq); } List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp); INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0}); INDArray exp1 = Nd4j.create(new float[]{0, 0, 1, 1, 1}); assertEquals(exp0, arr0); assertEquals(exp1, arr1); String json = tp.toJson(); TransformProcess tp2 = TransformProcess.fromJson(json); assertEquals(tp, tp2); List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp); INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get(); INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get(); assertEquals(exp0, arr0a); assertEquals(exp1, arr1a); }
@JsonCreator public GazeteerTransform(@JsonProperty("columnName") String columnName, @JsonProperty("newColumnName")String newColumnName, @JsonProperty("wordList") List<String> wordList) { super(columnName); this.newColumnName = newColumnName; this.wordList = wordList; this.gazeteer = new HashSet<>(wordList); }
GazeteerTransform extends BaseColumnTransform implements BagOfWordsTransform { @JsonCreator public GazeteerTransform(@JsonProperty("columnName") String columnName, @JsonProperty("newColumnName")String newColumnName, @JsonProperty("wordList") List<String> wordList) { super(columnName); this.newColumnName = newColumnName; this.wordList = wordList; this.gazeteer = new HashSet<>(wordList); } }
GazeteerTransform extends BaseColumnTransform implements BagOfWordsTransform { @JsonCreator public GazeteerTransform(@JsonProperty("columnName") String columnName, @JsonProperty("newColumnName")String newColumnName, @JsonProperty("wordList") List<String> wordList) { super(columnName); this.newColumnName = newColumnName; this.wordList = wordList; this.gazeteer = new HashSet<>(wordList); } @JsonCreator  GazeteerTransform(@JsonProperty("columnName") String columnName,
                             @JsonProperty("newColumnName")String newColumnName,
                             @JsonProperty("wordList") List<String> wordList); }
GazeteerTransform extends BaseColumnTransform implements BagOfWordsTransform { @JsonCreator public GazeteerTransform(@JsonProperty("columnName") String columnName, @JsonProperty("newColumnName")String newColumnName, @JsonProperty("wordList") List<String> wordList) { super(columnName); this.newColumnName = newColumnName; this.wordList = wordList; this.gazeteer = new HashSet<>(wordList); } @JsonCreator  GazeteerTransform(@JsonProperty("columnName") String columnName,
                             @JsonProperty("newColumnName")String newColumnName,
                             @JsonProperty("wordList") List<String> wordList); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override Writable map(Writable columnWritable); @Override Object mapSequence(Object sequence); @Override List<List<Writable>> mapSequence(List<List<Writable>> sequence); @Override String toString(); @Override Object map(Object input); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override long[] outputShape(); @Override List<String> vocabWords(); @Override INDArray transformFromObject(List<List<Object>> tokens); @Override INDArray transformFrom(List<List<Writable>> tokens); }
GazeteerTransform extends BaseColumnTransform implements BagOfWordsTransform { @JsonCreator public GazeteerTransform(@JsonProperty("columnName") String columnName, @JsonProperty("newColumnName")String newColumnName, @JsonProperty("wordList") List<String> wordList) { super(columnName); this.newColumnName = newColumnName; this.wordList = wordList; this.gazeteer = new HashSet<>(wordList); } @JsonCreator  GazeteerTransform(@JsonProperty("columnName") String columnName,
                             @JsonProperty("newColumnName")String newColumnName,
                             @JsonProperty("wordList") List<String> wordList); @Override ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType); @Override Writable map(Writable columnWritable); @Override Object mapSequence(Object sequence); @Override List<List<Writable>> mapSequence(List<List<Writable>> sequence); @Override String toString(); @Override Object map(Object input); @Override String outputColumnName(); @Override String[] outputColumnNames(); @Override String[] columnNames(); @Override String columnName(); @Override long[] outputShape(); @Override List<String> vocabWords(); @Override INDArray transformFromObject(List<List<Object>> tokens); @Override INDArray transformFrom(List<List<Writable>> tokens);  }