@Test public void testInferColumns() throws Exception { List<List<String>> categories = Arrays.asList( Arrays.asList("a","d") , Arrays.asList("b","e"), Arrays.asList("c","f") ); RecordReader listReader = new ListStringRecordReader(); listReader.initialize(new ListStringSplit(categories)); List<String> inferredSingle = TransformProcess.inferCategories(listReader,0); assertEquals(3,inferredSingle.size()); listReader.initialize(new ListStringSplit(categories)); Map<Integer, List<String>> integerListMap = TransformProcess.inferCategories(listReader, new int[]{0,1}); for(int i = 0; i < 2; i++) { assertEquals(3,integerListMap.get(i).size()); } }
public static List<String> inferCategories(RecordReader recordReader,int columnIndex) { Set<String> categories = new HashSet<>(); while(recordReader.hasNext()) { List<Writable> next = recordReader.next(); categories.add(next.get(columnIndex).toString()); } List<String> ret = new ArrayList<>(categories); Collections.sort(ret); return ret; }
TransformProcess implements Serializable { public static List<String> inferCategories(RecordReader recordReader,int columnIndex) { Set<String> categories = new HashSet<>(); while(recordReader.hasNext()) { List<Writable> next = recordReader.next(); categories.add(next.get(columnIndex).toString()); } List<String> ret = new ArrayList<>(categories); Collections.sort(ret); return ret; } }
TransformProcess implements Serializable { public static List<String> inferCategories(RecordReader recordReader,int columnIndex) { Set<String> categories = new HashSet<>(); while(recordReader.hasNext()) { List<Writable> next = recordReader.next(); categories.add(next.get(columnIndex).toString()); } List<String> ret = new ArrayList<>(categories); Collections.sort(ret); return ret; } TransformProcess(@JsonProperty("initialSchema") Schema initialSchema,
                            @JsonProperty("actionList") List<DataAction> actionList); private  TransformProcess(Builder builder); }
TransformProcess implements Serializable { public static List<String> inferCategories(RecordReader recordReader,int columnIndex) { Set<String> categories = new HashSet<>(); while(recordReader.hasNext()) { List<Writable> next = recordReader.next(); categories.add(next.get(columnIndex).toString()); } List<String> ret = new ArrayList<>(categories); Collections.sort(ret); return ret; } TransformProcess(@JsonProperty("initialSchema") Schema initialSchema,
                            @JsonProperty("actionList") List<DataAction> actionList); private  TransformProcess(Builder builder); List<DataAction> getActionList(); Schema getFinalSchema(); Schema getSchemaAfterStep(int step); List<Writable> execute(List<Writable> input); List<List<Writable>> executeSequenceToSequence(List<List<Writable>> input); List<List<Writable>> executeSequence(List<List<Writable>> inputSequence); List<List<List<Writable>>> executeToSequenceBatch(List<List<Writable>> inputExample); List<List<Writable>> executeToSequence(List<Writable> inputExample); List<Writable> executeSequenceToSingle(List<List<Writable>> inputSequence); String toJson(); String toYaml(); static TransformProcess fromJson(String json); static TransformProcess fromYaml(String yaml); static List<String> inferCategories(RecordReader recordReader,int columnIndex); static Map<Integer,List<String>> inferCategories(RecordReader recordReader,int[] columnIndices); List<List<Writable>> transformRawStringsToInputSequence(List<List<String>> sequence); List<Writable> transformRawStringsToInputList(List<String> values); List<Writable> transformRawStringsToInput(String... values); }
TransformProcess implements Serializable { public static List<String> inferCategories(RecordReader recordReader,int columnIndex) { Set<String> categories = new HashSet<>(); while(recordReader.hasNext()) { List<Writable> next = recordReader.next(); categories.add(next.get(columnIndex).toString()); } List<String> ret = new ArrayList<>(categories); Collections.sort(ret); return ret; } TransformProcess(@JsonProperty("initialSchema") Schema initialSchema,
                            @JsonProperty("actionList") List<DataAction> actionList); private  TransformProcess(Builder builder); List<DataAction> getActionList(); Schema getFinalSchema(); Schema getSchemaAfterStep(int step); List<Writable> execute(List<Writable> input); List<List<Writable>> executeSequenceToSequence(List<List<Writable>> input); List<List<Writable>> executeSequence(List<List<Writable>> inputSequence); List<List<List<Writable>>> executeToSequenceBatch(List<List<Writable>> inputExample); List<List<Writable>> executeToSequence(List<Writable> inputExample); List<Writable> executeSequenceToSingle(List<List<Writable>> inputSequence); String toJson(); String toYaml(); static TransformProcess fromJson(String json); static TransformProcess fromYaml(String yaml); static List<String> inferCategories(RecordReader recordReader,int columnIndex); static Map<Integer,List<String>> inferCategories(RecordReader recordReader,int[] columnIndices); List<List<Writable>> transformRawStringsToInputSequence(List<List<String>> sequence); List<Writable> transformRawStringsToInputList(List<String> values); List<Writable> transformRawStringsToInput(String... values);  }