WmfDumpFileManager { public List<MwDumpFile> findAllRelevantRevisionDumps(boolean preferCurrent) { MwDumpFile mainDump; if (preferCurrent) { mainDump = findMostRecentDump(DumpContentType.CURRENT); } else { mainDump = findMostRecentDump(DumpContentType.FULL); } if (mainDump == null) { return findAllDumps(DumpContentType.DAILY); } List<MwDumpFile> result = new ArrayList<>(); for (MwDumpFile dumpFile : findAllDumps(DumpContentType.DAILY)) { if (dumpFile.getDateStamp().compareTo(mainDump.getDateStamp()) > 0) { result.add(dumpFile); } } result.add(mainDump); if (logger.isInfoEnabled()) { StringBuilder logMessage = new StringBuilder(); logMessage.append("Found ") .append(result.size()) .append(" relevant dumps to process:"); for (MwDumpFile dumpFile : result) { logMessage.append("\n * ").append(dumpFile.toString()); } logger.info(logMessage.toString()); } return result; } WmfDumpFileManager(String projectName,
			DirectoryManager downloadDirectoryManager,
			WebResourceFetcher webResourceFetcher); List<MwDumpFile> findAllRelevantRevisionDumps(boolean preferCurrent); MwDumpFile findMostRecentDump(DumpContentType dumpContentType); List<MwDumpFile> findAllDumps(DumpContentType dumpContentType); static final String DOWNLOAD_DIRECTORY_NAME; }
@Test public void getAllRelevantDumps() throws IOException { wrf.setWebResourceContentsFromResource( "https: "/other-incr-wikidatawiki-index.html", this.getClass()); wrf.setWebResourceContentsFromResource( "https: "/wikidatawiki-index-old.html", this.getClass()); wrf.setWebResourceContentsFromResource( "https: "/wikidatawiki-20140210-md5sums.txt", this.getClass()); setLocalDump("20140220", DumpContentType.DAILY, true); setLocalDump("20140219", DumpContentType.FULL, true); setLocalDump("20140205", DumpContentType.DAILY, true); WmfDumpFileManager dumpFileManager = new WmfDumpFileManager( "wikidatawiki", dm, wrf); List<? extends MwDumpFile> dumpFiles = dumpFileManager .findAllRelevantRevisionDumps(true); String[] dumpDates = { "20140221", "20140220", "20140219", "20140218", "20140217", "20140216", "20140215", "20140214", "20140213", "20140212", "20140211", "20140210" }; boolean[] dumpIsLocal = { false, true, false, false, false, false, false, false, false, false, false, false }; assertEquals(dumpFiles.size(), dumpDates.length); for (int i = 0; i < dumpFiles.size(); i++) { if (i == dumpFiles.size() - 1) { assertEquals(dumpFiles.get(i).getDumpContentType(), DumpContentType.CURRENT); } else { assertEquals(dumpFiles.get(i).getDumpContentType(), DumpContentType.DAILY); } assertEquals(dumpFiles.get(i).getDateStamp(), dumpDates[i]); if (dumpIsLocal[i]) { assertTrue(dumpFiles.get(i) instanceof WmfLocalDumpFile); } else { assertFalse(dumpFiles.get(i) instanceof WmfLocalDumpFile); } } }
@Test public void getAllRelevantDumpsMainDumpMissing() throws IOException { setLocalDump("20140220", DumpContentType.DAILY, true); setLocalDump("20140210", DumpContentType.CURRENT, true); WmfDumpFileManager dumpFileManager = new WmfDumpFileManager( "wikidatawiki", dm, wrf); List<? extends MwDumpFile> dumpFiles = dumpFileManager .findAllRelevantRevisionDumps(false); assertEquals(dumpFiles.size(), 1); assertEquals(dumpFiles.get(0).getDumpContentType(), DumpContentType.DAILY); assertEquals(dumpFiles.get(0).getDateStamp(), "20140220"); assertTrue(dumpFiles.get(0) instanceof WmfLocalDumpFile); }
@Test public void processAllRelevantDumps() throws IOException { setLocalDump("20140221", DumpContentType.DAILY, true); setLocalDump("20140220", DumpContentType.DAILY, true); setLocalDump("20140219", DumpContentType.CURRENT, true); WmfDumpFileManager dumpFileManager = new WmfDumpFileManager( "wikidatawiki", dm, null); TestDumpfileProcessor dfp = new TestDumpfileProcessor(); for (MwDumpFile dumpFile : dumpFileManager .findAllRelevantRevisionDumps(true)) { dfp.processDumpFileContents(dumpFile.getDumpFileStream(), dumpFile); } assertEquals( dfp.result, "Contents of daily 20140221\nContents of daily 20140220\nContents of current 20140219\n"); }