DownloadUtils { public static void downloadResource(Odps odps, String projName, String resourceName, int limitDownloadRecordCount, char inputColumnSeperator) { if (odps == null || StringUtils.isBlank(projName) || StringUtils.isBlank(resourceName)) { throw new IllegalArgumentException("Missing arguments: odps|projName|resourceName"); } if (WareHouse.getInstance().getDownloadMode() == DownloadMode.NEVER) { throw new RuntimeException("Download resource '" + projName + "." + resourceName + "' Failed! Current download mode is:" + DownloadMode.NEVER + ".Please check parameter'" + Constants.LOCAL_DOWNLOAD_MODE + "'"); } try { if (!odps.resources().exists(projName, resourceName)) { throw new RuntimeException("Download resource: " + projName + "." + resourceName + " Failed!Remote resource not found! Download mode:" + WareHouse.getInstance().getDownloadMode()); } } catch (OdpsException e1) { throw new RuntimeException(e1); } Resource resource = odps.resources().get(projName, resourceName); File resFile = WareHouse.getInstance().getReourceFile(projName, resourceName); LOG.info("Start to download resource: " + resource.getName() + "-->" + resFile.getAbsolutePath() + ", type: " + resource.getType() + ",download mode:" + WareHouse.getInstance().getDownloadMode()); switch (resource.getType()) { case FILE: case JAR: case PY: case ARCHIVE: InputStream in = null; FileOutputStream out = null; try { in = odps.resources().getResourceAsStream(resource.getProject(), resource.getName()); resFile.getParentFile().mkdirs(); out = new FileOutputStream(resFile); IOUtils.copy(in, out); } catch (IOException e) { throw new RuntimeException(e); } catch (OdpsException e) { throw new RuntimeException(e); } finally { if (in != null) { try { in.close(); } catch (IOException e) { throw new RuntimeException(e); } } if (out != null) { try { out.close(); } catch (IOException e) { throw new RuntimeException(e); } } } break; case TABLE: TableResource tr = new TableResource(resource); Table stable = tr.getSourceTable(); String sourceProjName = stable.getProject(); String sourceTbleName = stable.getName(); PartitionSpec partSpec = tr.getSourceTablePartition(); TableInfo tableInfo = null; if (partSpec == null) { tableInfo = TableInfo.builder().projectName(sourceProjName).tableName(sourceTbleName) .build(); if (!WareHouse.getInstance().existsTable(sourceProjName, sourceTbleName)) { downloadTableSchemeAndData(odps, tableInfo, limitDownloadRecordCount, inputColumnSeperator); } } else { LinkedHashMap<String, String> partMap = PartitionUtils.convert(partSpec); tableInfo = TableInfo.builder().projectName(sourceProjName).tableName(sourceTbleName) .partSpec(partMap).build(); if (!WareHouse.getInstance().existsPartition(sourceProjName, sourceTbleName, partSpec)) { downloadTableSchemeAndData(odps, tableInfo, limitDownloadRecordCount, inputColumnSeperator); } } WareHouse.getInstance().createTableReourceFile(resource.getProject(), resource.getName(), tableInfo); break; default: break; } LOG.info("Finished download resource: " + resource.getName() + "-->" + resFile.getAbsolutePath() + ", type: " + resource.getType() + ",download mode:" + WareHouse.getInstance().getDownloadMode()); }  static void downloadTableSchemeAndData(Odps odps, TableInfo tableInfo,
                                                int limitDownloadRecordCount,
                                                char inputColumnSeperator); static File downloadTable(Odps odps, TableMeta tableMeta, PartitionSpec partition,
                                   int limitDownloadRecordCount, char inputColumnSeperator); static List<String[]> downloadTableData(Odps odps, String projectName, String tableName,
                                                 PartitionSpec partition,
                                                 int limitDownloadRecordCount, String[] readCols); static void downloadResource(Odps odps, String projName, String resourceName,
                                      int limitDownloadRecordCount, char inputColumnSeperator); static TableMeta downloadTableInfo(Odps odps, TableInfo tableInfo); static void validateTable(Odps odps, TableInfo tableInfo); static CsvReader newCsvReader(String fileName, char sep, Charset charset); static final Charset encoding; }
@Test public void testDownloadResource() throws IOException { WareHouse.getInstance().dropResourceIfExists(project, nonePartTableResource); WareHouse.getInstance().dropTableIfExists(project, nonePartTableResource); Assert.assertEquals(false, WareHouse.getInstance().existsTable(project, nonePartTableResource)); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, nonePartTableResource)); DownloadUtils.downloadResource(odps, project, nonePartTableResource, 10, ','); Assert.assertEquals(true, WareHouse.getInstance() .existsResource(project, nonePartTableResource)); Assert.assertEquals(true, WareHouse.getInstance().existsTable(project, nonePartTable)); WareHouse.getInstance().dropResourceIfExists(project, partTableResource); WareHouse.getInstance().dropTableIfExists(project, partTableResource); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, partTableResource)); Assert.assertEquals(false, WareHouse.getInstance().existsTable(project, partTableResource)); DownloadUtils.downloadResource(odps, project, partTableResource, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, partTableResource)); Assert.assertEquals(true, WareHouse.getInstance().existsTable(project, partTable)); }
@Test public void testResourceFile() throws IOException, OdpsException { String resourceName = "zhemin_res.file"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourceFile(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResourceFile(); }
@Test public void testResourceJar() throws OdpsException, IOException { String resourceName = "zhemin_res.jar"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourceJar(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResource(resourceName); }
@Test public void testResourcePy() throws OdpsException, IOException { String resourceName = "zhemin_res.py"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourcePy(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResource(resourceName); }
@Test public void testResourceArchive() throws OdpsException, IOException { String resourceName = "zhemin_res.tar.gz"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourceArchive(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResource(resourceName); }
@Test public void testResourceTable() throws OdpsException, IOException { String resourceName = "zhemin_res_src"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourceTable(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResource(resourceName); }
@Test public void testResourceTablePartition() throws OdpsException, IOException { String resourceName = "zhemin_res_pt"; WareHouse.getInstance().dropResourceIfExists(project, resourceName); Assert.assertEquals(false, WareHouse.getInstance().existsResource(project, resourceName)); addResourceTablePartition(); DownloadUtils.downloadResource(odps, project, resourceName, 10, ','); Assert.assertEquals(true, WareHouse.getInstance().existsResource(project, resourceName)); deleteResource(resourceName); }