@Test public void testDownloadTable() throws OdpsException, IOException { WareHouse.getInstance().dropTableIfExists(project, nonePartTable); Assert.assertEquals(false, WareHouse.getInstance().existsTable(project, nonePartTable)); TableInfo tableInfo = TableInfo.builder().projectName(project).tableName(nonePartTable).build(); DownloadUtils.downloadTableSchemeAndData(odps, tableInfo, 10, ','); List<Object[]> dataList = WareHouse.getInstance().readData(project, nonePartTable, null, null, ','); Assert.assertNotNull(dataList); Assert.assertEquals(1, dataList.size()); Object[] record = dataList.get(0); Assert.assertEquals(true, record[0] instanceof String); Assert.assertEquals("col2", (String) record[0]); Assert.assertEquals(true, record[1] instanceof Long); Assert.assertEquals(2L, ((Long) record[1]).longValue()); Assert.assertEquals(true, record[2] instanceof Double); Assert.assertEquals(true, ((Double) record[2]).doubleValue() == 2.2D); Assert.assertEquals(true, record[3] instanceof Boolean); Assert.assertEquals(false, (Boolean) record[3]); Assert.assertEquals(true, record[4] instanceof Date); Assert.assertEquals(BIG_DECIMAL, (BigDecimal) record[5]); Assert.assertEquals(true, record[5] instanceof BigDecimal); WareHouse.getInstance().dropTableIfExists(project, partTable); Assert.assertEquals(false, WareHouse.getInstance().existsTable(project, partTable)); tableInfo = TableInfo.builder().projectName(project).tableName(partTable) .partSpec(new PartitionSpec("p1=1,p2=2")).build(); DownloadUtils.downloadTableSchemeAndData(odps, tableInfo, 10, ','); dataList = WareHouse.getInstance().readData(project, partTable, new PartitionSpec("p1=1,p2=2"), null, ','); Assert.assertNotNull(dataList); Assert.assertEquals(1, dataList.size()); record = dataList.get(0); Assert.assertEquals(true, record[0] instanceof String); Assert.assertEquals("col2", (String) record[0]); Assert.assertEquals(true, record[1] instanceof Long); Assert.assertEquals(2L, ((Long) record[1]).longValue()); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, fileResource)); Assert.assertEquals(true, record[2] instanceof Double); Assert.assertEquals(true, ((Double) record[2]).doubleValue() == 2.2D); Assert.assertEquals(true, record[3] instanceof Boolean); Assert.assertEquals(false, (Boolean) record[3]); Assert.assertEquals(true, record[4] instanceof Date); Assert.assertEquals(BIG_DECIMAL, (BigDecimal) record[5]); Assert.assertEquals(true, record[5] instanceof BigDecimal); }
public static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition, int limitDownloadRecordCount, char inputColumnSeperator) { TableInfo tableInfo = TableInfo.builder().projectName(tableMeta.getProjName()) .tableName(tableMeta.getTableName()).partSpec(partition).build(); List<String[]> records = downloadTableData(odps, tableMeta.getProjName(), tableMeta.getTableName(), partition, limitDownloadRecordCount, null); File tableDir = WareHouse.getInstance().getTableDir(tableMeta.getProjName(), tableMeta.getTableName()); if (!tableDir.exists()) { tableDir.mkdirs(); } File dataDir = tableDir; if (partition != null) { dataDir = new File(tableDir, PartitionUtils.toString(partition)); if (!dataDir.exists()) { dataDir.mkdirs(); } } LOG.info("Start to write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); File dataFile = new File(dataDir, "data"); try { dataFile.createNewFile(); } catch (IOException e1) { } CsvWriter writer = new CsvWriter(dataFile.getAbsolutePath(), inputColumnSeperator, encoding); try { for (String[] record : records) { writer.writeRecord(record); } } catch (IOException e) { throw new RuntimeException(e); } writer.close(); LOG.info("Finished write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); return dataDir; }
DownloadUtils { public static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition, int limitDownloadRecordCount, char inputColumnSeperator) { TableInfo tableInfo = TableInfo.builder().projectName(tableMeta.getProjName()) .tableName(tableMeta.getTableName()).partSpec(partition).build(); List<String[]> records = downloadTableData(odps, tableMeta.getProjName(), tableMeta.getTableName(), partition, limitDownloadRecordCount, null); File tableDir = WareHouse.getInstance().getTableDir(tableMeta.getProjName(), tableMeta.getTableName()); if (!tableDir.exists()) { tableDir.mkdirs(); } File dataDir = tableDir; if (partition != null) { dataDir = new File(tableDir, PartitionUtils.toString(partition)); if (!dataDir.exists()) { dataDir.mkdirs(); } } LOG.info("Start to write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); File dataFile = new File(dataDir, "data"); try { dataFile.createNewFile(); } catch (IOException e1) { } CsvWriter writer = new CsvWriter(dataFile.getAbsolutePath(), inputColumnSeperator, encoding); try { for (String[] record : records) { writer.writeRecord(record); } } catch (IOException e) { throw new RuntimeException(e); } writer.close(); LOG.info("Finished write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); return dataDir; } }
DownloadUtils { public static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition, int limitDownloadRecordCount, char inputColumnSeperator) { TableInfo tableInfo = TableInfo.builder().projectName(tableMeta.getProjName()) .tableName(tableMeta.getTableName()).partSpec(partition).build(); List<String[]> records = downloadTableData(odps, tableMeta.getProjName(), tableMeta.getTableName(), partition, limitDownloadRecordCount, null); File tableDir = WareHouse.getInstance().getTableDir(tableMeta.getProjName(), tableMeta.getTableName()); if (!tableDir.exists()) { tableDir.mkdirs(); } File dataDir = tableDir; if (partition != null) { dataDir = new File(tableDir, PartitionUtils.toString(partition)); if (!dataDir.exists()) { dataDir.mkdirs(); } } LOG.info("Start to write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); File dataFile = new File(dataDir, "data"); try { dataFile.createNewFile(); } catch (IOException e1) { } CsvWriter writer = new CsvWriter(dataFile.getAbsolutePath(), inputColumnSeperator, encoding); try { for (String[] record : records) { writer.writeRecord(record); } } catch (IOException e) { throw new RuntimeException(e); } writer.close(); LOG.info("Finished write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); return dataDir; }  }
DownloadUtils { public static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition, int limitDownloadRecordCount, char inputColumnSeperator) { TableInfo tableInfo = TableInfo.builder().projectName(tableMeta.getProjName()) .tableName(tableMeta.getTableName()).partSpec(partition).build(); List<String[]> records = downloadTableData(odps, tableMeta.getProjName(), tableMeta.getTableName(), partition, limitDownloadRecordCount, null); File tableDir = WareHouse.getInstance().getTableDir(tableMeta.getProjName(), tableMeta.getTableName()); if (!tableDir.exists()) { tableDir.mkdirs(); } File dataDir = tableDir; if (partition != null) { dataDir = new File(tableDir, PartitionUtils.toString(partition)); if (!dataDir.exists()) { dataDir.mkdirs(); } } LOG.info("Start to write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); File dataFile = new File(dataDir, "data"); try { dataFile.createNewFile(); } catch (IOException e1) { } CsvWriter writer = new CsvWriter(dataFile.getAbsolutePath(), inputColumnSeperator, encoding); try { for (String[] record : records) { writer.writeRecord(record); } } catch (IOException e) { throw new RuntimeException(e); } writer.close(); LOG.info("Finished write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); return dataDir; }  static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo,
                                                int limitDownloadRecordCount,
                                                char inputColumnSeperator); static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition,
                                   int limitDownloadRecordCount, char inputColumnSeperator); static List<String[]> downloadTableData(Odps odps, String projectName, String tableName,
                                                 PartitionSpec partition,
                                                 int limitDownloadRecordCount, String[] readCols); static void downloadResource(Odps odps, String projName, String resourceName,
                                      int limitDownloadRecordCount, char inputColumnSeperator); static TableMeta downloadTableInfo(Odps odps, TableInfo tableInfo); static void validateTable(Odps odps, TableInfo tableInfo); static CsvReader newCsvReader(String fileName, char sep, Charset charset); }
DownloadUtils { public static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition, int limitDownloadRecordCount, char inputColumnSeperator) { TableInfo tableInfo = TableInfo.builder().projectName(tableMeta.getProjName()) .tableName(tableMeta.getTableName()).partSpec(partition).build(); List<String[]> records = downloadTableData(odps, tableMeta.getProjName(), tableMeta.getTableName(), partition, limitDownloadRecordCount, null); File tableDir = WareHouse.getInstance().getTableDir(tableMeta.getProjName(), tableMeta.getTableName()); if (!tableDir.exists()) { tableDir.mkdirs(); } File dataDir = tableDir; if (partition != null) { dataDir = new File(tableDir, PartitionUtils.toString(partition)); if (!dataDir.exists()) { dataDir.mkdirs(); } } LOG.info("Start to write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); File dataFile = new File(dataDir, "data"); try { dataFile.createNewFile(); } catch (IOException e1) { } CsvWriter writer = new CsvWriter(dataFile.getAbsolutePath(), inputColumnSeperator, encoding); try { for (String[] record : records) { writer.writeRecord(record); } } catch (IOException e) { throw new RuntimeException(e); } writer.close(); LOG.info("Finished write table: " + tableInfo.toString() + "-->" + dataDir.getAbsolutePath()); return dataDir; }  static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo,
                                                int limitDownloadRecordCount,
                                                char inputColumnSeperator); static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition,
                                   int limitDownloadRecordCount, char inputColumnSeperator); static List<String[]> downloadTableData(Odps odps, String projectName, String tableName,
                                                 PartitionSpec partition,
                                                 int limitDownloadRecordCount, String[] readCols); static void downloadResource(Odps odps, String projName, String resourceName,
                                      int limitDownloadRecordCount, char inputColumnSeperator); static TableMeta downloadTableInfo(Odps odps, TableInfo tableInfo); static void validateTable(Odps odps, TableInfo tableInfo); static CsvReader newCsvReader(String fileName, char sep, Charset charset); static final Charset encoding; }