TrecTextParser extends DocumentStreamParser { public Document nextDocument() throws IOException { String line; if (reader == null || null == waitFor("<DOC>")) { return null; } String identifier = parseDocNumber(); if (identifier == null) { return null; } StringBuilder buffer = new StringBuilder(); String[] startTags = {"<TEXT>", "<HEADLINE>", "<TITLE>", "<HL>", "<HEAD>", "<TTL>", "<DD>", "<DATE>", "<LP>", "<LEADPARA>" }; String[] endTags = {"</TEXT>", "</HEADLINE>", "</TITLE>", "</HL>", "</HEAD>", "</TTL>", "</DD>", "</DATE>", "</LP>", "</LEADPARA>" }; int inTag = -1; while ((line = reader.readLine()) != null) { if (line.startsWith("</DOC>")) { break; } if (line.startsWith("<")) { if (inTag >= 0 && line.startsWith(endTags[inTag])) { inTag = -1; buffer.append(line); buffer.append('\n'); } else if (inTag < 0) { for (int i = 0; i < startTags.length; i++) { if (line.startsWith(startTags[i])) { inTag = i; break; } } } } if (inTag >= 0) { buffer.append(line); buffer.append('\n'); } } return new Document(identifier, buffer.toString()); } TrecTextParser(DocumentSplit split, Parameters p); String waitFor(String tag); String parseDocNumber(); Document nextDocument(); @Override void close();  }
@Test public void testParseNothing() throws IOException { File f = FileUtility.createTemporary(); f.createNewFile(); try { DocumentSplit split = DocumentSplitFactory.file(f, "trectext"); TrecTextParser parser = new TrecTextParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }
@Test public void testAppTestGenDoc() throws IOException { String fileText = AppTest.trecDocument("CACM-0001", "This is some text in a document.\n"); File f = FileUtility.createTemporary(); try { StreamUtil.copyStringToFile(fileText, f); DocumentSplit split = DocumentSplitFactory.file(f, "trectext"); TrecTextParser parser = new TrecTextParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0001", document.name); assertEquals("<TEXT>\nThis is some text in a document.\n</TEXT>\n", document.text); document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }
@Test public void testDocumentStreamParser() throws IOException { String fileText = AppTest.trecDocument("CACM-0001", "This is some text in a document.\n"); File f = FileUtility.createTemporary(); try { StreamUtil.copyStringToFile(fileText, f); DocumentSplit split = DocumentSplitFactory.file(f, "trectext"); DocumentStreamParser parser = DocumentStreamParser.create(split, Parameters.create()); Document document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0001", document.name); assertEquals("<TEXT>\nThis is some text in a document.\n</TEXT>\n", document.text); document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }
@Test public void testParseOneDocument() throws IOException { String fileText = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<TEXT>\n" + "This is some text in a document.\n" + "</TEXT>\n" + "</DOC>\n"; File f = FileUtility.createTemporary(); try { StreamUtil.copyStringToFile(fileText, f); DocumentSplit split = DocumentSplitFactory.file(f); TrecTextParser parser = new TrecTextParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0001", document.name); assertEquals("<TEXT>\nThis is some text in a document.\n</TEXT>\n", document.text); document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }
@Test public void testParseTwoDocuments() throws IOException { String fileText = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<TEXT>\n" + "This is some text in a document.\n" + "</TEXT>\n" + "</DOC>\n" + "<DOC>\n" + "<DOCNO>CACM-0002</DOCNO>\n" + "<TEXT>\n" + "This is some text in a document.\n" + "</TEXT>\n" + "</DOC>\n"; File f = FileUtility.createTemporary(); try { StreamUtil.copyStringToFile(fileText, f); DocumentSplit split = DocumentSplitFactory.file(f); TrecTextParser parser = new TrecTextParser(split, Parameters.create()); Document document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0001", document.name); assertEquals("<TEXT>\nThis is some text in a document.\n</TEXT>\n", document.text); document = parser.nextDocument(); assertNotNull(document); assertEquals("CACM-0002", document.name); assertEquals("<TEXT>\nThis is some text in a document.\n</TEXT>\n", document.text); document = parser.nextDocument(); assertNull(document); } finally { f.delete(); } }