TrecWebParser extends DocumentStreamParser { public Document nextDocument() throws IOException { String line = null; if (null == waitFor("<DOC>")) { close(); return null; } String identifier = waitFor("<DOCNO>"); identifier = identifier.substring(7).trim(); identifier = identifier.substring(0, identifier.length() - 8); identifier = new String(identifier.trim()); waitFor("<DOCHDR>"); String url = readUrl(); waitFor("</DOCHDR>"); StringBuilder buffer = new StringBuilder(20 * 1024); while ((line = reader.readLine()) != null) { if (line.startsWith("</DOC>")) { break; } buffer.append(line); buffer.append('\n'); } Document result = new Document(identifier, buffer.toString()); result.metadata.put("url", new String(url)); result.metadata.put("identifier", result.name); return result; } TrecWebParser(DocumentSplit split, Parameters p); String waitFor(String tag); void close(); String scrubUrl(String url); String readUrl(); Document nextDocument();  }
@Test public void testParseNothing() throws IOException { File f = FileUtility.createTemporary(); f.createNewFile(); try { DocumentSplit split = DocumentSplitFactory.file(f); TrecWebParser parser = new TrecWebParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }
@Test public void testParseOneDocument() throws IOException { String fileText = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<DOCHDR>\n" + "http: + "even more text in this part\n" + "</DOCHDR>\n" + "This is some text in a document.\n" + "</DOC>\n"; File f = FileUtility.createTemporary(); try { StreamUtil.copyStringToFile(fileText, f); DocumentSplit split = DocumentSplitFactory.file(f); TrecWebParser parser = new TrecWebParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0001", document.name); assertEquals("http: assertEquals("This is some text in a document.\n", document.text); document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }