@Test public void testDownloader() throws IOException, InterruptedException { URL url = new URL("http: File tmp1 = File.createTempFile("downloader-test", ".txt"); FileDownloader downloader = new FileDownloader(); downloader.download(url, tmp1); assertTrue(tmp1.isFile()); List<String> lines = FileUtils.readLines(tmp1); assert(lines.size() > 10); assertTrue(lines.get(0).startsWith("User-agent:")); File tmp2 = File.createTempFile("downloader-test", ".txt"); FileUtils.copyURLToFile(url, tmp2); assertTrue(tmp2.isFile()); assertTrue(FileUtils.readFileToString(tmp2).startsWith("User-agent:")); assertEquals(FileUtils.readFileToString(tmp1), FileUtils.readFileToString(tmp2)); }
public File download(URL url, File file) throws InterruptedException { LOG.info("beginning download of " + url + " to " + file); for (int i=1; i <= maxAttempts; i++) { try { AtomicBoolean stop = new AtomicBoolean(false); DownloadInfo info = new DownloadInfo(url); DownloadMonitor monitor = new DownloadMonitor(info); info.extract(stop, monitor); file.getParentFile().mkdirs(); WGet wget = new WGet(info, file); wget.download(stop, monitor); LOG.info("Download complete: " + file.getAbsolutePath()); while (!monitor.isFinished()) { Thread.sleep(sleepTime); } return file; } catch (DownloadIOCodeError e) { if (i < maxAttempts) { LOG.info("Failed to download " + url + ". Reconnecting in " + (i * backoffTime / 1000) + " seconds (HTTP " + e.getCode() + "-Error " + url + ")"); Thread.sleep(backoffTime * i); } else { LOG.warn("Failed to download " + file + " (HTTP " + e.getCode() + "-Error " + url + ")"); } } } return null; }
FileDownloader { public File download(URL url, File file) throws InterruptedException { LOG.info("beginning download of " + url + " to " + file); for (int i=1; i <= maxAttempts; i++) { try { AtomicBoolean stop = new AtomicBoolean(false); DownloadInfo info = new DownloadInfo(url); DownloadMonitor monitor = new DownloadMonitor(info); info.extract(stop, monitor); file.getParentFile().mkdirs(); WGet wget = new WGet(info, file); wget.download(stop, monitor); LOG.info("Download complete: " + file.getAbsolutePath()); while (!monitor.isFinished()) { Thread.sleep(sleepTime); } return file; } catch (DownloadIOCodeError e) { if (i < maxAttempts) { LOG.info("Failed to download " + url + ". Reconnecting in " + (i * backoffTime / 1000) + " seconds (HTTP " + e.getCode() + "-Error " + url + ")"); Thread.sleep(backoffTime * i); } else { LOG.warn("Failed to download " + file + " (HTTP " + e.getCode() + "-Error " + url + ")"); } } } return null; } }
FileDownloader { public File download(URL url, File file) throws InterruptedException { LOG.info("beginning download of " + url + " to " + file); for (int i=1; i <= maxAttempts; i++) { try { AtomicBoolean stop = new AtomicBoolean(false); DownloadInfo info = new DownloadInfo(url); DownloadMonitor monitor = new DownloadMonitor(info); info.extract(stop, monitor); file.getParentFile().mkdirs(); WGet wget = new WGet(info, file); wget.download(stop, monitor); LOG.info("Download complete: " + file.getAbsolutePath()); while (!monitor.isFinished()) { Thread.sleep(sleepTime); } return file; } catch (DownloadIOCodeError e) { if (i < maxAttempts) { LOG.info("Failed to download " + url + ". Reconnecting in " + (i * backoffTime / 1000) + " seconds (HTTP " + e.getCode() + "-Error " + url + ")"); Thread.sleep(backoffTime * i); } else { LOG.warn("Failed to download " + file + " (HTTP " + e.getCode() + "-Error " + url + ")"); } } } return null; } FileDownloader(); }
FileDownloader { public File download(URL url, File file) throws InterruptedException { LOG.info("beginning download of " + url + " to " + file); for (int i=1; i <= maxAttempts; i++) { try { AtomicBoolean stop = new AtomicBoolean(false); DownloadInfo info = new DownloadInfo(url); DownloadMonitor monitor = new DownloadMonitor(info); info.extract(stop, monitor); file.getParentFile().mkdirs(); WGet wget = new WGet(info, file); wget.download(stop, monitor); LOG.info("Download complete: " + file.getAbsolutePath()); while (!monitor.isFinished()) { Thread.sleep(sleepTime); } return file; } catch (DownloadIOCodeError e) { if (i < maxAttempts) { LOG.info("Failed to download " + url + ". Reconnecting in " + (i * backoffTime / 1000) + " seconds (HTTP " + e.getCode() + "-Error " + url + ")"); Thread.sleep(backoffTime * i); } else { LOG.warn("Failed to download " + file + " (HTTP " + e.getCode() + "-Error " + url + ")"); } } } return null; } FileDownloader(); File download(URL url, File file); void setSleepTime(int sleepTime); void setMaxAttempts(int maxAttempts); void setDisplayInfo(int displayInfo); void setBackoffTime(int backoffTime); }
FileDownloader { public File download(URL url, File file) throws InterruptedException { LOG.info("beginning download of " + url + " to " + file); for (int i=1; i <= maxAttempts; i++) { try { AtomicBoolean stop = new AtomicBoolean(false); DownloadInfo info = new DownloadInfo(url); DownloadMonitor monitor = new DownloadMonitor(info); info.extract(stop, monitor); file.getParentFile().mkdirs(); WGet wget = new WGet(info, file); wget.download(stop, monitor); LOG.info("Download complete: " + file.getAbsolutePath()); while (!monitor.isFinished()) { Thread.sleep(sleepTime); } return file; } catch (DownloadIOCodeError e) { if (i < maxAttempts) { LOG.info("Failed to download " + url + ". Reconnecting in " + (i * backoffTime / 1000) + " seconds (HTTP " + e.getCode() + "-Error " + url + ")"); Thread.sleep(backoffTime * i); } else { LOG.warn("Failed to download " + file + " (HTTP " + e.getCode() + "-Error " + url + ")"); } } } return null; } FileDownloader(); File download(URL url, File file); void setSleepTime(int sleepTime); void setMaxAttempts(int maxAttempts); void setDisplayInfo(int displayInfo); void setBackoffTime(int backoffTime); static final Logger LOG; }