FileDownloader { public void download() throws IOException, TunnelException { if (sh != null) { String msg = String.format("file [" + id + "] start"); sh.log(msg); System.err.println(sim.format(new Date()) + " - " + msg); } String fd = DshipContext.INSTANCE.get(Constants.FIELD_DELIMITER); String rd = DshipContext.INSTANCE.get(Constants.RECORD_DELIMITER); String ni = DshipContext.INSTANCE.get(Constants.NULL_INDICATOR); String dfp = DshipContext.INSTANCE.get(Constants.DATE_FORMAT_PATTERN); String tz = DshipContext.INSTANCE.get(Constants.TIME_ZONE); String charset = DshipContext.INSTANCE.get(Constants.CHARSET); boolean exponential = false; String e = DshipContext.INSTANCE.get(Constants.EXPONENTIAL); if (e != null && e.equalsIgnoreCase("true")) { exponential = true; } if (isCsv) { writer = new CsvRecordWriter(file, charset); } else { writer = new TextRecordWriter(file, fd, rd); } TableSchema schema = ds.getSchema(); RecordConverter converter = new RecordConverter(schema, ni, dfp, tz, charset, exponential); if ("true".equalsIgnoreCase(DshipContext.INSTANCE.get(Constants.HEADER))) { writeHeader(writer, schema); } preTime = System.currentTimeMillis(); DshipRecordReader recordReader = ds.getRecordReader(start, end); long count = 0; Record r; while ((r = readAndTime(recordReader)) != null) { writeAndTime(writer, converter.format(r)); count++; currTime = System.currentTimeMillis(); if (currTime - preTime > 5000) { printProgress(count); preTime = currTime; } ODPSConsoleUtils.checkThreadInterrupted(); } writer.close(); writtenBytes = writer.getWrittedBytes(); if (sh != null) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append(String.format("file [%d] OK. total: %s", id, Util.toReadableBytes(writtenBytes))); messageBuilder.append(localIOStopWatch.getFormattedSummary()); messageBuilder.append(tunnelIOStopWatch.getFormattedSummary()); System.err.println(sim.format(new Date()) + " - " + messageBuilder.toString()); sh.log(messageBuilder.toString()); } } FileDownloader(String path, Long id, Long start, Long end, TunnelDownloadSession ds, SessionHistory sh);  FileDownloader(String path, Long id, Long start, Long end, TunnelDownloadSession ds, SessionHistory sh, boolean isCsv); void download(); long getWrittenBytes();  }
@Test public void testDownloadSlice() throws Exception { String[] args = new String[]{"download", projectName + "." + TEST_TABLE_NAME + "/ds='2113',pt='pttest'", "src/test/resources/file/filedownloader/tmp.txt", "-fd=||", "-rd=\n", "-dfp=yyyyMMddHHmmss"}; OptionsBuilder.buildDownloadOption(args); MockDownloadSession us = new MockDownloadSession(); us.crash = 0; us.alreadCrash = 0; FileDownloader sd = new FileDownloader("src/test/resources/file/filedownloader/tmp.txt", 0L, 0L, 10L, us, null); sd.download(); assertEquals("download", readFile("src/test/resources/file/filedownloader/sample.txt"), readFile("src/test/resources/file/filedownloader/tmp.txt")); }
@Test public void testDownloadSlice_SelecteColumns() throws Exception { String[] args = new String[]{"download", projectName + "." + TEST_TABLE_NAME + "/ds='2113',pt='pttest'", "src/test/resources/file/filedownloader/tmp_partial.txt", "-fd=||", "-rd=\n", "-dfp=yyyyMMddHHmmss"}; OptionsBuilder.buildDownloadOption(args); MockDownloadSession us = new MockDownloadSession(); us.crash = 0; us.alreadCrash = 0; List<Column> columns = us.getSchema().getColumns(); columns.remove(2); TableSchema schema = new TableSchema(); schema.setColumns(columns); us.setSchema(schema); FileDownloader sd = new FileDownloader("src/test/resources/file/filedownloader/tmp_partial.txt", 0L, 0L, 10L, us, null); sd.download(); assertEquals("download", readFile("src/test/resources/file/filedownloader/sample_partial.txt"), readFile("src/test/resources/file/filedownloader/tmp_partial.txt")); }
@Test public void testDownloadFileWithCrash4_Success() throws Exception { String[] args = new String[] {"download", projectName + "." + TEST_TABLE_NAME + "/ds='2113',pt='pttest'", "src/test/resources/file/filedownloader/tmp.txt", "-fd=||", "-rd=\n", "-dfp=yyyyMMddHHmmss"}; OptionsBuilder.buildDownloadOption(args); MockDownloadSession us = new MockDownloadSession(); MockDownloadSession.crash = 4; FileDownloader sd = new FileDownloader("src/test/resources/file/filedownloader/tmp.txt", 0L, 0L, 10L, us, null); sd.download(); assertEquals("download", readFile("src/test/resources/file/filedownloader/sample.txt"), readFile("src/test/resources/file/filedownloader/tmp.txt")); }
@Test public void testDownloadFileWithCrash5_Fail() throws Exception { String[] args = new String[] {"download", projectName + "." + TEST_TABLE_NAME + "/ds='2113',pt='pttest'", "src/test/resources/file/filedownloader/tmp.txt", "-fd=||", "-rd=\n", "-dfp=yyyyMMddHHmmss"}; OptionsBuilder.buildDownloadOption(args); MockDownloadSession us = new MockDownloadSession(); MockDownloadSession.crash = 5; FileDownloader sd = new FileDownloader("src/test/resources/file/filedownloader/tmp.txt", 0L, 0L, 10L, us, null); try { sd.download(); } catch (Exception e) { assertEquals("error message", "ERROR: download read error, retry exceed 5.\ncrash :5", e.getMessage()); } }