OrcFile extends Scheme<Configuration, RecordReader, OutputCollector, Corc, Corc> { public static SourceBuilder source() { return new SourceBuilder(); } OrcFile(StructTypeInfo typeInfo, SearchArgument searchArgument, Fields fields, StructTypeInfo schemaTypeInfo,
      ConverterFactory converterFactory);  OrcFile(Fields fields, StructTypeInfo schemaTypeInfo, ConverterFactory converterFactory); private  OrcFile(StructTypeInfo typeInfo, SearchArgument searchArgument, Fields fields, StructTypeInfo schemaTypeInfo,
      ConverterFactory converterFactory, SchemeType type); static SourceBuilder source(); static SinkBuilder sink(); @Override void sourceConfInit(FlowProcess<? extends Configuration> flowProcess,
      Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf); @Override void sourcePrepare(FlowProcess<? extends Configuration> flowProcess, SourceCall<Corc, RecordReader> sourceCall); @Override boolean source(FlowProcess<? extends Configuration> flowProcess, SourceCall<Corc, RecordReader> sourceCall); @Override void sinkConfInit(FlowProcess<? extends Configuration> flowProcess,
      Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf); @Override void sinkPrepare(FlowProcess<? extends Configuration> flowProcess, SinkCall<Corc, OutputCollector> sinkCall); @SuppressWarnings("unchecked") @Override void sink(FlowProcess<? extends Configuration> flowProcess, SinkCall<Corc, OutputCollector> sinkCall); @Override boolean isSymmetrical(); @Override boolean isSource(); @Override boolean isSink(); static final Fields ROW__ID; static final boolean IGNORE_ROW_ID; }
@Test(expected = IllegalArgumentException.class) public void missingFieldsTypes() { OrcFile.source().declaredFields(new Fields("A")).schemaFromFile().build(); }
@Test(expected = IllegalArgumentException.class) public void nullFieldsTypes() { OrcFile.source().declaredFields(new Fields("A", null)).schemaFromFile().build(); }
@Test(expected = IllegalArgumentException.class) public void duplicateFields() { Fields upperA = new Fields("A", String.class); Fields lowerA = new Fields("a", String.class); OrcFile.source().declaredFields(upperA.append(lowerA)).schemaFromFile().build(); }
@Test public void writeTypical() throws IOException { Data data = new DataBuilder(FIELDS_AB).addTuple("A1", "B1").addTuple("A2", "B2").build(); Tap<?, ?, ?> tap = new Hfs(OrcFile.source().declaredFields(FIELDS_AB).schemaFromFile().build(), path); Plunger.writeData(data).toTap(tap); try (OrcReader reader = getOrcReader()) { assertThat(reader.hasNext(), is(true)); List<Object> list = reader.next(); assertThat(list.size(), is(2)); assertThat(list.get(0), is((Object) "A1")); assertThat(list.get(1), is((Object) "B1")); assertThat(reader.hasNext(), is(true)); list = reader.next(); assertThat(list.size(), is(2)); assertThat(list.get(0), is((Object) "A2")); assertThat(list.get(1), is((Object) "B2")); assertThat(reader.hasNext(), is(false)); } }
@Test public void readTypical() throws IOException { try (OrcWriter writer = new OrcWriter.Builder(conf, new Path(path, "part-00000")) .addField("a", TypeInfoFactory.stringTypeInfo) .addField("b", TypeInfoFactory.stringTypeInfo) .build()) { writer.addRow("A1", "B1"); writer.addRow("A2", "B2"); } List<TupleEntry> actual = Plunger.readDataFromTap( new Hfs(OrcFile.source().declaredFields(FIELDS_AB).schemaFromFile().build(), path)).asTupleEntryList(); List<TupleEntry> expected = new DataBuilder(FIELDS_AB) .addTuple("A1", "B1") .addTuple("A2", "B2") .build() .asTupleEntryList(); assertThat(actual, is(tupleEntryList(expected))); }
@Test public void readColumnProjectionA() throws IOException { try (OrcWriter writer = new OrcWriter.Builder(conf, new Path(path, "part-00000")) .addField("a", TypeInfoFactory.stringTypeInfo) .addField("b", TypeInfoFactory.stringTypeInfo) .build()) { writer.addRow("A1", "B1"); writer.addRow("A2", "B2"); } List<TupleEntry> actual = Plunger.readDataFromTap( new Hfs(OrcFile.source().declaredFields(FIELD_A).schemaFromFile().build(), path)).asTupleEntryList(); List<TupleEntry> expected = new DataBuilder(FIELD_A).addTuple("A1").addTuple("A2").build().asTupleEntryList(); assertThat(actual, is(tupleEntryList(expected))); }
@Test public void readColumnProjectionB() throws IOException { OrcWriter.Builder builder = new OrcWriter.Builder(conf, new Path(path, "part-00000")).addField("a", TypeInfoFactory.stringTypeInfo).addField("b", TypeInfoFactory.stringTypeInfo); try (OrcWriter writer = builder.build()) { writer.addRow("A1", "B1"); writer.addRow("A2", "B2"); } List<TupleEntry> actual = Plunger.readDataFromTap( new Hfs(OrcFile.source().declaredFields(FIELD_B).schemaFromFile().build(), path)).asTupleEntryList(); List<TupleEntry> expected = new DataBuilder(FIELD_B).addTuple("B1").addTuple("B2").build().asTupleEntryList(); assertThat(actual, is(tupleEntryList(expected))); }
@Test public void readMissing() throws IOException { try (OrcWriter writer = new OrcWriter.Builder(conf, new Path(path, "part-00000")).addField("a", TypeInfoFactory.stringTypeInfo).build()) { writer.addRow("A1"); writer.addRow("A2"); } List<TupleEntry> actual = Plunger.readDataFromTap( new Hfs(OrcFile.source().declaredFields(FIELDS_AB).schemaFromFile().build(), path)).asTupleEntryList(); List<TupleEntry> expected = new DataBuilder(FIELDS_AB) .addTuple("A1", null) .addTuple("A2", null) .build() .asTupleEntryList(); assertThat(actual, is(tupleEntryList(expected))); }
@Test public void writeViaTypeInfo() throws IOException { StructTypeInfo typeInfo = new StructTypeInfoBuilder() .add("a", TypeInfoFactory.stringTypeInfo) .add("b", TypeInfoFactory.getListTypeInfo(TypeInfoFactory.stringTypeInfo)) .add("c", TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo)) .add("d", TypeInfoFactory.getCharTypeInfo(1)) .add("e", TypeInfoFactory.getVarcharTypeInfo(1)) .add("f", TypeInfoFactory.getDecimalTypeInfo(2, 1)) .build(); Fields fields = SchemaFactory.newFields(typeInfo); Map<String, String> map = new HashMap<>(); map.put("C1", "C1"); Data data = new DataBuilder(fields) .addTuple("A1", Arrays.asList("B1"), map, "x", "y", new BigDecimal("1.234")) .build(); Tap<?, ?, ?> tap = new Hfs(OrcFile.source().columns(typeInfo).schemaFromFile().build(), path); Plunger.writeData(data).toTap(tap); try (OrcReader reader = getOrcReader()) { assertThat(reader.hasNext(), is(true)); List<Object> list = reader.next(); assertThat(list.size(), is(6)); assertThat(list.get(0), is((Object) "A1")); assertThat(list.get(1), is((Object) Arrays.asList("B1"))); assertThat(list.get(2), is((Object) map)); assertThat(((HiveChar) list.get(3)).getValue(), is((Object) "x")); assertThat(((HiveVarchar) list.get(4)).getValue(), is((Object) "y")); assertThat(((HiveDecimal) list.get(5)).bigDecimalValue(), is((Object) new BigDecimal("1.2"))); assertThat(reader.hasNext(), is(false)); } }
@Test public void readStringPredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.stringTypeInfo; try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow("hello"); writer.addRow("world"); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", TypeInfoFactory.stringTypeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory.newBuilder().startAnd().equals("a", PredicateLeaf.Type.STRING, "hello").end().build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat(list.get(0).getObject(0), is((Object) "hello")); }
@Test public void readBooleanPredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.booleanTypeInfo; try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow(true); writer.addRow(false); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", TypeInfoFactory.booleanTypeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory.newBuilder().startAnd().equals("a", PredicateLeaf.Type.BOOLEAN, true).end().build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat(list.get(0).getObject(0), is((Object) true)); }
@Test public void readDatePredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.dateTypeInfo; Date date1 = Date.valueOf("1970-01-01"); Date date2 = Date.valueOf("1970-01-02"); try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow(date1); writer.addRow(date2); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", TypeInfoFactory.dateTypeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory .newBuilder() .startAnd() .equals("a", PredicateLeaf.Type.DATE, date1) .end() .build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat((list.get(0).getObject(0)), is((Object)date1)); }
@Ignore("Below runs into an issue where Timestamp object is re-used when Tuples from file are iterated over") @Test public void readTimestampPredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.timestampTypeInfo; Timestamp timestamp1 = Timestamp.valueOf("1970-01-01 00:00:00"); Timestamp timestamp2 = Timestamp.valueOf("1970-01-02 00:00:00"); Timestamp timestamp3 = Timestamp.valueOf("1971-01-02 00:00:00"); try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow(timestamp1); writer.addRow(timestamp2); writer.addRow(timestamp3); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", TypeInfoFactory.timestampTypeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory .newBuilder() .startAnd() .equals("a", PredicateLeaf.Type.TIMESTAMP, timestamp1) .end() .build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat(((Timestamp) list.get(0).getObject(0)), is(timestamp1)); }
@Test(expected=UnsupportedOperationException.class) public void readTimestampNotSupported() throws IOException { Timestamp timestamp1 = Timestamp.valueOf("1970-01-01 00:00:00"); SearchArgument searchArgument = SearchArgumentFactory .newBuilder() .startAnd() .equals("a", PredicateLeaf.Type.TIMESTAMP, timestamp1) .end() .build(); OrcFile.source().searchArgument(searchArgument).build(); }
@Test public void readDecimalPredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(2, 1); try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow(HiveDecimal.create("0.0")); writer.addRow(HiveDecimal.create("0.1")); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", typeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory .newBuilder() .startAnd() .equals("a", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("0.1")) .end() .build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat(list.get(0).getObject(0), is((Object) new BigDecimal("0.1"))); }
@Test public void readFloatPredicatePushdown() throws IOException { TypeInfo typeInfo = TypeInfoFactory.floatTypeInfo; try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow(0.0f); writer.addRow(0.1f); } StructTypeInfo structTypeInfo = new StructTypeInfoBuilder().add("a", typeInfo).build(); SearchArgument searchArgument = SearchArgumentFactory .newBuilder() .startAnd() .equals("a", PredicateLeaf.Type.FLOAT, 0.0) .end() .build(); OrcFile orcFile = OrcFile.source().columns(structTypeInfo).schemaFromFile().searchArgument(searchArgument).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); List<Tuple> list = Plunger.readDataFromTap(tap).asTupleList(); assertThat(list.size(), is(1)); assertThat(list.get(0).getObject(0), is((Object) 0.0f)); }
@Test(expected = TupleException.class) public void readIncorrectType() throws IOException { TypeInfo typeInfo = TypeInfoFactory.stringTypeInfo; try (OrcWriter writer = getOrcWriter(typeInfo)) { writer.addRow("hello"); } Fields intField = new Fields("A", int.class); OrcFile orcFile = OrcFile.source().declaredFields(intField).schema(intField).build(); Tap<?, ?, ?> tap = new Hfs(orcFile, path); Plunger.readDataFromTap(tap); }