HarvestLinksFn extends AppFunction { @Override public void run(Parameters p, PrintStream output) throws Exception { if (!p.isString("inputPath") && !p.isList("inputPath", String.class)) { output.println(getHelpString()); return; } Job job = getHarvestLinksJob(p); if (job != null) { TupleflowAppUtil.runTupleFlowJob(job, p, output); } output.println("Done harvesting link."); }  @Override String getName(); @Override String getHelpString(); @Override void run(Parameters p, PrintStream output);  }
@Test public void testHarvestLinks() throws Exception { File tempDir = FileUtility.createTemporaryDirectory(); try { File input = new File(tempDir, "input.trecweb"); writeInput(input); File indri = new File(tempDir, "indri"); File galago = new File(tempDir, "galago"); File jobTmp = new File(tempDir, "jobTmp"); Parameters p = Parameters.create(); p.set("inputPath", input.getAbsolutePath()); p.set("indri", true); p.set("filePrefix", tempDir.getAbsolutePath()); p.set("prefixReplacement", indri.getAbsolutePath()); p.set("galago", true); p.set("outputFolder", galago.getAbsolutePath()); p.set("galagoDist", 3); p.set("distrib", 2); p.set("galagoJobDir", jobTmp.getAbsolutePath()); p.set("server", false); HarvestLinksFn hl = new HarvestLinksFn(); hl.run(p, System.out); assert (indri.exists() && indri.list().length == 1) : "There should be one file in the indri directory."; File outputFile = new File(indri, "input.trecweb"); assert (outputFile.exists()) : "input.trecweb should exist in the indri directory."; GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(outputFile)); String data = StreamUtil.copyStreamToString(gzis); String expectedPrefix = "DOCNO=test-0\n" + "http: + "LINKS=12\n" + "LINKDOCNO=test-0\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-1\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-2\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-4\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-6\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-6\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-6\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-6\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-6\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-7\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-8\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "LINKDOCNO=test-11\n" + "LINKFROM=http: + "TEXT=\"t.0\"\n" + "DOCNO=test-1\n" + "http: + "LINKS=6\n" + "LINKDOCNO=test-0\n" + "LINKFROM=http: assert (data.startsWith(expectedPrefix)) : "input.trecweb should match the provided prefix.\n" + data; File names = new File(galago, "names"); File srcOrder = new File(galago, "srcNameOrder"); File destOrder = new File(galago, "destNameOrder"); assert (names.isDirectory() && srcOrder.isDirectory() && destOrder.isDirectory()) : "galago folder should contain three folders."; File[] nameFiles = FileUtility.safeListFiles(names); assertEquals(nameFiles.length, 3); TypeReader<DocumentUrl> reader1 = OrderedCombiner.combineFromFileObjs(Arrays.asList(nameFiles), new DocumentUrl.IdentifierOrder()); DocumentUrl durl = reader1.read(); long count = 0; while (durl != null) { count += 1; durl = reader1.read(); } assertEquals(count, 12); TypeReader<ExtractedLink> reader2 = OrderedCombiner.combineFromFileObjs(Arrays.asList(FileUtility.safeListFiles(srcOrder)), new ExtractedLink.SrcNameOrder()); ExtractedLink ln = reader2.read(); count = 0; while (ln != null) { count += 1; ln = reader2.read(); } assertEquals(count, 53); } finally { FSUtil.deleteDirectory(tempDir); } }