ForwardIndexWriter extends KeyValueWriter<KeyValuePair> { @Override public void close () throws IOException { Parameters manifest = writer.getManifest (); manifest.set ("fwindexStatistics/docsInCollection", this.docsInCollection); manifest.set ("fwindexStatistics/totalTermsInCollection", this.totalTermsInCollection); manifest.set ("fwindexStatistics/maxTermFreqInCollection", this.maxTermFreqInCollection); super.close (); } ForwardIndexWriter(TupleFlowParameters parameters); @Override GenericElement prepare(KeyValuePair kvp); @Override void close(); static void verify(TupleFlowParameters parameters, ErrorStore store);  }
@Test public void testWriteThenRead() throws Exception { Map<Long, DocTermsInfo> docTermsInfoHM = new HashMap<>(); Map<Long, DocTermsInfo> trueData = new HashMap<>(); File tmp = File.createTempFile("forwardIndexWriterTest", ".tmp"); tmp.deleteOnExit(); if (!tmp.exists()) { tmp.createNewFile(); } String fileName = tmp.getAbsolutePath(); Parameters p = Parameters.create(); p.set("filename", fileName); p.set("fwindexStatistics/docsInCollection", 0L); p.set("fwindexStatistics/totalTermsInCollection", 0L); p.set("fwindexStatistics/maxTermFreqInCollection", 0L); try { ForwardIndexWriter writer = new ForwardIndexWriter(new FakeParameters(p)); for (int i = 0; i < terms.length; i++) { DocTermsInfo dti = new DocTermsInfo((long) i); dti.docUniqueTermCount = uniqueTermCnt[i]; dti.docTermCount = terms[i].length; dti.docMaxTermFreq = maxTF[i]; for (int j = 0; j < terms[i].length; j++) { DocTermsInfo.TermInfo ti = new DocTermsInfo.TermInfo(); ti.term = terms[i][j]; ti.termFreq = tf[i][j]; int begPos = positions[i][j]; int begOffs = begoffs[i][j]; DocTermsInfo.PositionInfo pi = new DocTermsInfo.PositionInfo(begPos, begOffs); ti.positionInfoList.add(pi); dti.termsInfoHM.put(ti.term, ti); } trueData.put((long) i, dti); byte[] keyBytes = Utility.fromLong((long) i); byte[] valueBytes = ForwardIndexSerializer.toBytes(dti); KeyValuePair kvp = new KeyValuePair(keyBytes, valueBytes); writer.process(kvp); docsInCollection++; totalTermsInCollection += dti.docTermCount; maxTermFreqInCollection = Math.max(maxTermFreqInCollection, dti.docMaxTermFreq); p.set("fwindexStatistics/docsInCollection", docsInCollection); p.set("fwindexStatistics/totalTermsInCollection", totalTermsInCollection); p.set("fwindexStatistics/maxTermFreqInCollection", maxTermFreqInCollection); } writer.close(); ForwardIndexReader reader = new ForwardIndexReader(fileName); Parameters manifest = reader.getManifest(); String tmp_manifestString = manifest.toString(); String manifestString = tmp_manifestString.replace("\\\\", "\\"); String manifestStringTruth = "{ \"blockCount\" : 1 , \"blockSize\" : 16383 , " + "\"emptyIndexFile\" : false , " + "\"filename\" : \"" + fileName + "\" , " + "\"fwindexStatistics/docsInCollection\" : 3 , " + "\"fwindexStatistics/maxTermFreqInCollection\" : 1 , " + "\"fwindexStatistics/totalTermsInCollection\" : 12 , " + "\"keyCount\" : 3 , \"maxKeySize\" : 16383 , " + "\"readerClass\" : \"org.lemurproject.galago.contrib.index.disk.ForwardIndexReader\" , " + "\"writerClass\" : \"org.lemurproject.galago.contrib.index.disk.ForwardIndexWriter\" }"; assertEquals("Written and expected manifests differ.", manifestString, manifestStringTruth); ForwardIndexReader.KeyIterator ki = reader.getIterator(); while (!ki.isDone()) { long id = Long.parseLong(ki.getKeyString()); DocTermsInfo dti = (DocTermsInfo) ForwardIndexSerializer.fromBytes(ki.getValueBytes()); DocTermsInfo dtiTrue = (DocTermsInfo) trueData.get(id); assertEquals(dti.docUniqueTermCount, dtiTrue.docUniqueTermCount); assertEquals(dti.docTermCount, dtiTrue.docTermCount); assertEquals(dti.docMaxTermFreq, dtiTrue.docMaxTermFreq); Map<String, DocTermsInfo.TermInfo> tiHM = dti.termsInfoHM; Map<String, DocTermsInfo.TermInfo> trueTiHM = dtiTrue.termsInfoHM; assertEquals(tiHM.size(), trueTiHM.size()); Set ks = tiHM.entrySet(); Set trueKs = trueTiHM.entrySet(); assertEquals(ks.size(), trueKs.size()); assertTrue("Ks Set CONTAINS all True Keys", trueKs.containsAll(trueKs)); assertFalse("Ks Set DOES NOT CONTAIN all True Keys", !trueKs.containsAll(trueKs)); Iterator itr = ks.iterator(); while (itr.hasNext()) { Map.Entry me = (Map.Entry) itr.next(); String term = (String) me.getKey(); DocTermsInfo.TermInfo ti = (DocTermsInfo.TermInfo) me.getValue(); ArrayList<DocTermsInfo.PositionInfo> piList = ti.positionInfoList; DocTermsInfo.TermInfo trueTi = trueTiHM.get(term); ArrayList<DocTermsInfo.PositionInfo> truePiList = trueTi.positionInfoList; StringBuffer sb = new StringBuffer(); StringBuffer trueSb = new StringBuffer(); String tiStr = DocTermsInfo.toString(ti); String trueTiStr = DocTermsInfo.toString(trueTi); sb.append("Doc: " + id + " " + term + " --> " + tiStr); trueSb.append("Doc: " + id + " " + term + " --> " + trueTiStr); String sbStr = sb.toString(); String trueSbStr = trueSb.toString(); assertEquals(piList.size(), truePiList.size()); assertEquals(sbStr, trueSbStr); } ki.nextKey(); } reader.close(); } catch (Exception ex) { ex.printStackTrace(); } }