@Test(expected = RuntimeException.class) public void testDownloadTableWithWrongPartSpec() throws IOException { WareHouse.getInstance().dropTableIfExists(project, partTable); Assert.assertEquals(false, WareHouse.getInstance().existsTable(project, partTable)); TableInfo tableInfo = TableInfo.builder().projectName(project).tableName(partTable) .partSpec(new PartitionSpec("p1=2")).build(); DownloadUtils.downloadTableSchemeAndData(odps, tableInfo, 10, ','); }
public static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || tableInfo == null) { throw new IllegalArgumentException("Missing arguments:odps|tableInfo"); } if (StringUtils.isBlank(tableInfo.getProjectName())) { tableInfo.setProjectName(odps.getDefaultProject()); } validateTable(odps, tableInfo); Table table = odps.tables().get(tableInfo.getProjectName(), tableInfo.getTableName()); File tableDir = WareHouse.getInstance().getTableDir(tableInfo.getProjectName(), tableInfo.getTableName()); if (!tableDir.exists()){ tableDir.mkdirs(); } TableMeta tableMeta = TableMeta.fromTable(table); SchemaUtils.generateSchemaFile(tableMeta, null, tableDir); if (tableMeta.getPartitions() != null && tableMeta.getPartitions().length > 0) { LinkedHashMap<String, String> parts = tableInfo.getPartSpec(); int downloadedPartition = 0; for (Partition partition : table.getPartitions()) { PartitionSpec spec = partition.getPartitionSpec(); if (matches(spec, parts)) { downloadTable(odps, tableMeta, spec, limitDownloadRecordCount, inputColumnSeperator); downloadedPartition++; } } if (downloadedPartition == 0) { throw new RuntimeException("No partition found for " + tableInfo.toString()); } } else { if (tableInfo.getPartSpec().size() > 0) { throw new RuntimeException("Warehouse not exists table: " + tableInfo + " ,while try to download from remote server found it is not a partitioned table"); } downloadTable(odps, tableMeta, null, limitDownloadRecordCount, inputColumnSeperator); } }
DownloadUtils { public static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || tableInfo == null) { throw new IllegalArgumentException("Missing arguments:odps|tableInfo"); } if (StringUtils.isBlank(tableInfo.getProjectName())) { tableInfo.setProjectName(odps.getDefaultProject()); } validateTable(odps, tableInfo); Table table = odps.tables().get(tableInfo.getProjectName(), tableInfo.getTableName()); File tableDir = WareHouse.getInstance().getTableDir(tableInfo.getProjectName(), tableInfo.getTableName()); if (!tableDir.exists()){ tableDir.mkdirs(); } TableMeta tableMeta = TableMeta.fromTable(table); SchemaUtils.generateSchemaFile(tableMeta, null, tableDir); if (tableMeta.getPartitions() != null && tableMeta.getPartitions().length > 0) { LinkedHashMap<String, String> parts = tableInfo.getPartSpec(); int downloadedPartition = 0; for (Partition partition : table.getPartitions()) { PartitionSpec spec = partition.getPartitionSpec(); if (matches(spec, parts)) { downloadTable(odps, tableMeta, spec, limitDownloadRecordCount, inputColumnSeperator); downloadedPartition++; } } if (downloadedPartition == 0) { throw new RuntimeException("No partition found for " + tableInfo.toString()); } } else { if (tableInfo.getPartSpec().size() > 0) { throw new RuntimeException("Warehouse not exists table: " + tableInfo + " ,while try to download from remote server found it is not a partitioned table"); } downloadTable(odps, tableMeta, null, limitDownloadRecordCount, inputColumnSeperator); } } }
DownloadUtils { public static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || tableInfo == null) { throw new IllegalArgumentException("Missing arguments:odps|tableInfo"); } if (StringUtils.isBlank(tableInfo.getProjectName())) { tableInfo.setProjectName(odps.getDefaultProject()); } validateTable(odps, tableInfo); Table table = odps.tables().get(tableInfo.getProjectName(), tableInfo.getTableName()); File tableDir = WareHouse.getInstance().getTableDir(tableInfo.getProjectName(), tableInfo.getTableName()); if (!tableDir.exists()){ tableDir.mkdirs(); } TableMeta tableMeta = TableMeta.fromTable(table); SchemaUtils.generateSchemaFile(tableMeta, null, tableDir); if (tableMeta.getPartitions() != null && tableMeta.getPartitions().length > 0) { LinkedHashMap<String, String> parts = tableInfo.getPartSpec(); int downloadedPartition = 0; for (Partition partition : table.getPartitions()) { PartitionSpec spec = partition.getPartitionSpec(); if (matches(spec, parts)) { downloadTable(odps, tableMeta, spec, limitDownloadRecordCount, inputColumnSeperator); downloadedPartition++; } } if (downloadedPartition == 0) { throw new RuntimeException("No partition found for " + tableInfo.toString()); } } else { if (tableInfo.getPartSpec().size() > 0) { throw new RuntimeException("Warehouse not exists table: " + tableInfo + " ,while try to download from remote server found it is not a partitioned table"); } downloadTable(odps, tableMeta, null, limitDownloadRecordCount, inputColumnSeperator); } }  }
DownloadUtils { public static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || tableInfo == null) { throw new IllegalArgumentException("Missing arguments:odps|tableInfo"); } if (StringUtils.isBlank(tableInfo.getProjectName())) { tableInfo.setProjectName(odps.getDefaultProject()); } validateTable(odps, tableInfo); Table table = odps.tables().get(tableInfo.getProjectName(), tableInfo.getTableName()); File tableDir = WareHouse.getInstance().getTableDir(tableInfo.getProjectName(), tableInfo.getTableName()); if (!tableDir.exists()){ tableDir.mkdirs(); } TableMeta tableMeta = TableMeta.fromTable(table); SchemaUtils.generateSchemaFile(tableMeta, null, tableDir); if (tableMeta.getPartitions() != null && tableMeta.getPartitions().length > 0) { LinkedHashMap<String, String> parts = tableInfo.getPartSpec(); int downloadedPartition = 0; for (Partition partition : table.getPartitions()) { PartitionSpec spec = partition.getPartitionSpec(); if (matches(spec, parts)) { downloadTable(odps, tableMeta, spec, limitDownloadRecordCount, inputColumnSeperator); downloadedPartition++; } } if (downloadedPartition == 0) { throw new RuntimeException("No partition found for " + tableInfo.toString()); } } else { if (tableInfo.getPartSpec().size() > 0) { throw new RuntimeException("Warehouse not exists table: " + tableInfo + " ,while try to download from remote server found it is not a partitioned table"); } downloadTable(odps, tableMeta, null, limitDownloadRecordCount, inputColumnSeperator); } }  static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo,
                                                int limitDownloadRecordCount,
                                                char inputColumnSeperator); static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition,
                                   int limitDownloadRecordCount, char inputColumnSeperator); static List<String[]> downloadTableData(Odps odps, String projectName, String tableName,
                                                 PartitionSpec partition,
                                                 int limitDownloadRecordCount, String[] readCols); static void downloadResource(Odps odps, String projName, String resourceName,
                                      int limitDownloadRecordCount, char inputColumnSeperator); static TableMeta downloadTableInfo(Odps odps, TableInfo tableInfo); static void validateTable(Odps odps, TableInfo tableInfo); static CsvReader newCsvReader(String fileName, char sep, Charset charset); }
DownloadUtils { public static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || tableInfo == null) { throw new IllegalArgumentException("Missing arguments:odps|tableInfo"); } if (StringUtils.isBlank(tableInfo.getProjectName())) { tableInfo.setProjectName(odps.getDefaultProject()); } validateTable(odps, tableInfo); Table table = odps.tables().get(tableInfo.getProjectName(), tableInfo.getTableName()); File tableDir = WareHouse.getInstance().getTableDir(tableInfo.getProjectName(), tableInfo.getTableName()); if (!tableDir.exists()){ tableDir.mkdirs(); } TableMeta tableMeta = TableMeta.fromTable(table); SchemaUtils.generateSchemaFile(tableMeta, null, tableDir); if (tableMeta.getPartitions() != null && tableMeta.getPartitions().length > 0) { LinkedHashMap<String, String> parts = tableInfo.getPartSpec(); int downloadedPartition = 0; for (Partition partition : table.getPartitions()) { PartitionSpec spec = partition.getPartitionSpec(); if (matches(spec, parts)) { downloadTable(odps, tableMeta, spec, limitDownloadRecordCount, inputColumnSeperator); downloadedPartition++; } } if (downloadedPartition == 0) { throw new RuntimeException("No partition found for " + tableInfo.toString()); } } else { if (tableInfo.getPartSpec().size() > 0) { throw new RuntimeException("Warehouse not exists table: " + tableInfo + " ,while try to download from remote server found it is not a partitioned table"); } downloadTable(odps, tableMeta, null, limitDownloadRecordCount, inputColumnSeperator); } }  static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo,
                                                int limitDownloadRecordCount,
                                                char inputColumnSeperator); static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition,
                                   int limitDownloadRecordCount, char inputColumnSeperator); static List<String[]> downloadTableData(Odps odps, String projectName, String tableName,
                                                 PartitionSpec partition,
                                                 int limitDownloadRecordCount, String[] readCols); static void downloadResource(Odps odps, String projName, String resourceName,
                                      int limitDownloadRecordCount, char inputColumnSeperator); static TableMeta downloadTableInfo(Odps odps, TableInfo tableInfo); static void validateTable(Odps odps, TableInfo tableInfo); static CsvReader newCsvReader(String fileName, char sep, Charset charset); static final Charset encoding; }