ArrowConverter { public static List<FieldVector> toArrowColumns(final BufferAllocator bufferAllocator, final Schema schema, List<List<Writable>> dataVecRecord) { int numRows = dataVecRecord.size(); List<FieldVector> ret = createFieldVectors(bufferAllocator,schema,numRows); for(int j = 0; j < schema.numColumns(); j++) { FieldVector fieldVector = ret.get(j); int row = 0; for(List<Writable> record : dataVecRecord) { Writable writable = record.get(j); setValue(schema.getType(j),fieldVector,writable,row); row++; } } return ret; }  static INDArray toArray(ArrowWritableRecordTimeSeriesBatch arrowWritableRecordBatch); static INDArray toArray(ArrowWritableRecordBatch arrowWritableRecordBatch); static INDArray convertArrowVector(FieldVector fieldVector,ColumnType type); static List<FieldVector> convertToArrowVector(INDArray from,List<String> name,ColumnType type,BufferAllocator bufferAllocator); static void writeRecordBatchTo(List<List<Writable>> recordBatch, Schema inputSchema,OutputStream outputStream); static void writeRecordBatchTo(BufferAllocator bufferAllocator ,List<List<Writable>> recordBatch, Schema inputSchema,OutputStream outputStream); static List<List<List<Writable>>> toArrowWritablesTimeSeries(List<FieldVector> fieldVectors,Schema schema,int timeSeriesLength); static ArrowWritableRecordBatch toArrowWritables(List<FieldVector> fieldVectors,Schema schema); static List<Writable> toArrowWritablesSingle(List<FieldVector> fieldVectors,Schema schema); static Pair<Schema,ArrowWritableRecordBatch> readFromFile(FileInputStream input); static Pair<Schema,ArrowWritableRecordBatch> readFromFile(File input); static Pair<Schema,ArrowWritableRecordBatch> readFromBytes(byte[] input); static org.apache.arrow.vector.types.pojo.Schema toArrowSchema(Schema schema); static Schema toDatavecSchema(org.apache.arrow.vector.types.pojo.Schema schema); static Field field(String name,ArrowType arrowType); static Field getFieldForColumn(String name,ColumnType columnType); static Field doubleField(String name); static Field floatField(String name); static Field intField(String name); static Field longField(String name); static Field stringField(String name); static Field booleanField(String name); static DictionaryProvider providerForVectors(List<FieldVector> vectors,List<Field> fields); static List<FieldVector> toArrowColumns(final BufferAllocator bufferAllocator, final Schema schema, List<List<Writable>> dataVecRecord); static List<FieldVector> toArrowColumnsTimeSeries(final BufferAllocator bufferAllocator,
                                                              final Schema schema,
                                                              List<List<List<Writable>>> dataVecRecord); static List<FieldVector> toArrowColumnsTimeSeriesHelper(final BufferAllocator bufferAllocator,
                                                                        final Schema schema,
                                                                        List<List<List<T>>> dataVecRecord); static List<FieldVector> toArrowColumnsStringSingle(final BufferAllocator bufferAllocator, final Schema schema, List<String> dataVecRecord); static List<FieldVector> toArrowColumnsStringTimeSeries(final BufferAllocator bufferAllocator,
                                                                    final Schema schema,
                                                                    List<List<List<String>>> dataVecRecord); static List<FieldVector> toArrowColumnsString(final BufferAllocator bufferAllocator, final Schema schema, List<List<String>> dataVecRecord); static void setValue(ColumnType columnType,FieldVector fieldVector,Object value,int row); static TimeStampMilliVector vectorFor(BufferAllocator allocator,String name,Date[] data); static TimeStampMilliVector timeVectorOf(BufferAllocator allocator,String name,int length); static VarCharVector vectorFor(BufferAllocator allocator,String name,String[] data); static VarCharVector stringVectorOf(BufferAllocator allocator,String name,int length); static Float4Vector vectorFor(BufferAllocator allocator,String name,float[] data); static Float4Vector floatVectorOf(BufferAllocator allocator,String name,int length); static Float8Vector vectorFor(BufferAllocator allocator,String name,double[] data); static Float8Vector doubleVectorOf(BufferAllocator allocator,String name,int length); static BitVector vectorFor(BufferAllocator allocator,String name,boolean[] data); static BitVector booleanVectorOf(BufferAllocator allocator,String name,int length); static IntVector vectorFor(BufferAllocator allocator,String name,int[] data); static IntVector intVectorOf(BufferAllocator allocator,String name,int length); static BigIntVector vectorFor(BufferAllocator allocator,String name,long[] data); static BigIntVector longVectorOf(BufferAllocator allocator,String name,int length); static Writable fromEntry(int item,FieldVector from,ColumnType columnType);  }
@Test public void testArrowBatchSetTime() { Schema.Builder schema = new Schema.Builder(); List<String> single = new ArrayList<>(); for(int i = 0; i < 2; i++) { schema.addColumnTime(String.valueOf(i),TimeZone.getDefault()); single.add(String.valueOf(i)); } List<List<Writable>> input = Arrays.asList( Arrays.<Writable>asList(new LongWritable(0),new LongWritable(1)), Arrays.<Writable>asList(new LongWritable(2),new LongWritable(3)) ); List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator,schema.build(),input); ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector,schema.build()); List<Writable> assertion = Arrays.<Writable>asList(new LongWritable(4), new LongWritable(5)); writableRecordBatch.set(1, Arrays.<Writable>asList(new LongWritable(4),new LongWritable(5))); List<Writable> recordTest = writableRecordBatch.get(1); assertEquals(assertion,recordTest); }
@Test public void testArrowBatchSet() { Schema.Builder schema = new Schema.Builder(); List<String> single = new ArrayList<>(); for(int i = 0; i < 2; i++) { schema.addColumnInteger(String.valueOf(i)); single.add(String.valueOf(i)); } List<List<Writable>> input = Arrays.asList( Arrays.<Writable>asList(new IntWritable(0),new IntWritable(1)), Arrays.<Writable>asList(new IntWritable(2),new IntWritable(3)) ); List<FieldVector> fieldVector = ArrowConverter.toArrowColumns(bufferAllocator,schema.build(),input); ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector,schema.build()); List<Writable> assertion = Arrays.<Writable>asList(new IntWritable(4), new IntWritable(5)); writableRecordBatch.set(1, Arrays.<Writable>asList(new IntWritable(4),new IntWritable(5))); List<Writable> recordTest = writableRecordBatch.get(1); assertEquals(assertion,recordTest); }