DumpDocTermsFn extends AppFunction { @Override public void run(Parameters p, PrintStream output) throws Exception { assert (p.isString("index")) : "dump-doc-terms requires an 'index' paramter."; assert (p.containsKey("iidList") || p.containsKey("eidList")) : "dump-doc-terms requires an 'iidList' and/or 'eidList' parameter."; String eid = null; long iid = 0L; if (!p.containsKey("index") || (!p.containsKey("iidList") && !p.containsKey("eidList"))) { output.println("No index defined or missing a doc ID list."); output.println(this.getHelpString()); return; } String index = p.getString("index"); Stemmer stemmer = null; if (index.endsWith("porter")) { stemmer = new Porter2Stemmer(); } if (index.endsWith("krovetz")) { stemmer = new KrovetzStemmer(); } int lastSlash = index.lastIndexOf(File.separator); String indexRoot = index.substring(0, lastSlash); Map<Long, String> iid2eidTM = new TreeMap<>(); if (p.containsKey("iidList")) { String idsStr = p.getString("iidList"); String[] toks = idsStr.split("[ ,]+"); int len = toks.length; if (len > 0) { DiskNameReader dnr = new DiskNameReader(indexRoot + File.separator + "names"); for (String id : toks) { iid = Long.parseLong(id); eid = dnr.getDocumentName(iid); if (!iid2eidTM.containsKey(iid)) { iid2eidTM.put(iid, eid); } } dnr.close(); } } if (p.containsKey("eidList")) { long badIDCounter = 0; String idsStr = p.getString("eidList"); String[] toks = idsStr.split("[ ,]+"); int len = toks.length; if (len > 0) { DiskNameReverseReader dnrr = new DiskNameReverseReader(indexRoot + File.separator + "names.reverse"); for (String eidStr : toks) { iid = dnrr.getDocumentIdentifier(eidStr); if (iid == -1) { iid = -1 - badIDCounter; badIDCounter++; } if (!iid2eidTM.containsKey(iid)) { iid2eidTM.put(iid, eidStr); } } dnrr.close(); } } LocalRetrieval retrieval = new LocalRetrieval(indexRoot, Parameters.create()); for (Map.Entry<Long, String> entry : iid2eidTM.entrySet()) { iid = entry.getKey(); eid = entry.getValue(); if (iid < 0) { output.println("Doc: ---\t[" + eid + "]\tDoc ID does not exist"); } else if (eid == null) { output.println("Doc: " + iid + "\t[ --- ]\tDoc ID does not exist"); } else { Document doc = retrieval.getDocument(eid, new Document.DocumentComponents(true, true, true)); Map<String, List<Integer>> termPos = doc.getTermPositions(stemmer); StringBuilder sb = new StringBuilder(); long maxTF = 0; for (Map.Entry<String, List<Integer>> entry2 : termPos.entrySet()) { String term = entry2.getKey(); List<Integer> pos = entry2.getValue(); sb.append(term + "," + iid); if (pos.size() > maxTF) { maxTF = pos.size(); } for (Integer i : pos) { sb.append("," + i); } sb.append("\n"); } output.println("Doc: " + iid + " [" + eid + "]\tTerm Count: " + termPos.size() + "\tTotal Words: " + doc.terms.size() + "\tMax TF: " + maxTF); output.println(sb.toString()); } } }  static void main(String[] args); @Override String getName(); @Override String getHelpString(); @Override void run(Parameters p, PrintStream output); static final Logger logger; }
@Test public void testSingleIID() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--iidList=1,"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 22); String[] expectedLines = {"Doc: 1 [AP890101-0002]\tTerm Count: 21\tTotal Words: 29\tMax TF: 3", "\ta,1,4,13", "\tamerican,1,25", "\tat,1,16", "\tbusiness,1,26", "\tdoc,1,1", "\terects,1,3", "\tfactory,1,5,15", "\tfor,1,9", "\tfuture,1,8,23", "\tin,1,12", "\tis,1,27", "\tminiature,1,14", "\tmissouri,1,20", "\tnow,1,28", "\tof,1,6,19,24", "\trolla,1,21", "\tsecond,1,0", "\tstudents,1,10", "\tthe,1,7,17,22", "\tuniversity,1,2,18", "\tworking,1,11" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertTrue("Output line [" + i + "] doesn't match expected", expectedL.equals(outputL)); i++; } }
@Test public void testSingleEID() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--eidList=AP890101-0002"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 22); String[] expectedLines = {"Doc: 1 [AP890101-0002]\tTerm Count: 21\tTotal Words: 29\tMax TF: 3", "\ta,1,4,13", "\tamerican,1,25", "\tat,1,16", "\tbusiness,1,26", "\tdoc,1,1", "\terects,1,3", "\tfactory,1,5,15", "\tfor,1,9", "\tfuture,1,8,23", "\tin,1,12", "\tis,1,27", "\tminiature,1,14", "\tmissouri,1,20", "\tnow,1,28", "\tof,1,6,19,24", "\trolla,1,21", "\tsecond,1,0", "\tstudents,1,10", "\tthe,1,7,17,22", "\tuniversity,1,2,18", "\tworking,1,11" }; int i = 0; for (String line : expectedLines) { String outputL = outputLines[i].trim(); String expectedL = line.trim(); assertTrue("Output line [" + i + "] doesn't match expected", expectedL.equals(outputL)); i++; } }
@Test public void testTwoIIDs() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--iidList=0,2"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Expected 45 output lines but got " + outputLines.length, outputLines.length == 45); String[] expectedLines = {"Doc: 0 [AP890101-0001]\tTerm Count: 22\tTotal Words: 23\tMax TF: 2", "\t1960s,0,22", "\t60s,0,2", "\ta,0,13", "\tare,0,4", "\tbeen,0,10", "\tcelluloid,0,7", "\tdoc,0,1", "\tfilmmakers,0,16", "\tfilms,0,3", "\tfirst,0,0", "\tgeneration,0,15", "\tgrew,0,18", "\thas,0,9", "\there,0,5", "\tin,0,20", "\tnew,0,14", "\tpassed,0,11", "\tthe,0,6,21", "\tto,0,12", "\ttorch,0,8", "\tup,0,19", "\twho,0,17", "", "Doc: 2 [AP890104-0235]\tTerm Count: 20\tTotal Words: 25\tMax TF: 2", "\t10,2,7,20", "\t1990s,2,24", "\t300,2,17", "\ta,2,16", "\tchrysler,2,3,9", "\tcorp,2,11", "\tdoc,2,2", "\tduring,2,22", "\tengine,2,8,21", "\thorsepower,2,18", "\tit,2,13", "\tmake,2,15", "\tmotors,2,10", "\tproduce,2,5", "\tsays,2,12", "\tthe,2,0,23", "\tthird,2,1", "\tto,2,4", "\tv,2,6,19", "\twill,2,14" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals(expectedL, outputL); i++; } }
@Test public void testTwoEIDs() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--eidList=AP890101-0001,AP890104-0235"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 45); String[] expectedLines = {"Doc: 0 [AP890101-0001]\tTerm Count: 22\tTotal Words: 23\tMax TF: 2", "\t1960s,0,22", "\t60s,0,2", "\ta,0,13", "\tare,0,4", "\tbeen,0,10", "\tcelluloid,0,7", "\tdoc,0,1", "\tfilmmakers,0,16", "\tfilms,0,3", "\tfirst,0,0", "\tgeneration,0,15", "\tgrew,0,18", "\thas,0,9", "\there,0,5", "\tin,0,20", "\tnew,0,14", "\tpassed,0,11", "\tthe,0,6,21", "\tto,0,12", "\ttorch,0,8", "\tup,0,19", "\twho,0,17", "", "Doc: 2 [AP890104-0235]\tTerm Count: 20\tTotal Words: 25\tMax TF: 2", "\t10,2,7,20", "\t1990s,2,24", "\t300,2,17", "\ta,2,16", "\tchrysler,2,3,9", "\tcorp,2,11", "\tdoc,2,2", "\tduring,2,22", "\tengine,2,8,21", "\thorsepower,2,18", "\tit,2,13", "\tmake,2,15", "\tmotors,2,10", "\tproduce,2,5", "\tsays,2,12", "\tthe,2,0,23", "\tthird,2,1", "\tto,2,4", "\tv,2,6,19", "\twill,2,14" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals(expectedL, outputL); i++; } }
@Test public void testUndefIID() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--iidList=999,"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 1); String[] expectedLines = {"Doc: 999\t[ --- ]\tDoc ID does not exist"}; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertTrue("Output line [" + i + "] doesn't match expected", expectedL.equals(outputL)); i++; } }
@Test public void testUndefEID() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--eidList=XYZ"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 1); String[] expectedLines = {"Doc: ---\t[XYZ]\tDoc ID does not exist"}; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertTrue("Output line [" + i + "] doesn't match expected", expectedL.equals(outputL)); i++; } }
@Test public void testIIDandEIDwithOverlap() throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(byteArrayStream); App.run(new String[]{"dump-doc-terms", "--index=" + childPath.getAbsolutePath(), "--iidList=0,1", "--eidList=AP890104-0235,AP890101-0002"}, printStream); String tmp_output = byteArrayStream.toString(); String output = tmp_output.replace("\r", ""); String[] outputLines = output.split("\n"); assertTrue("Output should contain " + outputLines.length, outputLines.length == 68); String[] expectedLines = {"Doc: 0 [AP890101-0001]\tTerm Count: 22\tTotal Words: 23\tMax TF: 2", "\t1960s,0,22", "\t60s,0,2", "\ta,0,13", "\tare,0,4", "\tbeen,0,10", "\tcelluloid,0,7", "\tdoc,0,1", "\tfilmmakers,0,16", "\tfilms,0,3", "\tfirst,0,0", "\tgeneration,0,15", "\tgrew,0,18", "\thas,0,9", "\there,0,5", "\tin,0,20", "\tnew,0,14", "\tpassed,0,11", "\tthe,0,6,21", "\tto,0,12", "\ttorch,0,8", "\tup,0,19", "\twho,0,17", "", "Doc: 1 [AP890101-0002]\tTerm Count: 21\tTotal Words: 29\tMax TF: 3", "\ta,1,4,13", "\tamerican,1,25", "\tat,1,16", "\tbusiness,1,26", "\tdoc,1,1", "\terects,1,3", "\tfactory,1,5,15", "\tfor,1,9", "\tfuture,1,8,23", "\tin,1,12", "\tis,1,27", "\tminiature,1,14", "\tmissouri,1,20", "\tnow,1,28", "\tof,1,6,19,24", "\trolla,1,21", "\tsecond,1,0", "\tstudents,1,10", "\tthe,1,7,17,22", "\tuniversity,1,2,18", "\tworking,1,11", "", "Doc: 2 [AP890104-0235]\tTerm Count: 20\tTotal Words: 25\tMax TF: 2", "\t10,2,7,20", "\t1990s,2,24", "\t300,2,17", "\ta,2,16", "\tchrysler,2,3,9", "\tcorp,2,11", "\tdoc,2,2", "\tduring,2,22", "\tengine,2,8,21", "\thorsepower,2,18", "\tit,2,13", "\tmake,2,15", "\tmotors,2,10", "\tproduce,2,5", "\tsays,2,12", "\tthe,2,0,23", "\tthird,2,1", "\tto,2,4", "\tv,2,6,19", "\twill,2,14" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals(expectedL, outputL); i++; } }