DocumentSource implements ExNihiloSource<DocumentSplit> { public static List<DocumentSplit> processZipFile(File fp, Parameters conf) throws IOException { String forceFileType = conf.get("filetype", (String) null); ArrayList<DocumentSplit> splits = new ArrayList<>(); try (ZipFile zipF = ZipUtil.open(fp)) { List<String> names = ZipUtil.listZipFile(zipF); for (String name : names) { String fileType = forceFileType; if (fileType == null) { File inside = new File(name); String extension = FSUtil.getExtension(inside); if (DocumentStreamParser.hasParserForExtension(extension)) { fileType = extension; } else { fileType = detectTrecTextOrWeb(ZipUtil.streamZipEntry(zipF, name), fp.getAbsolutePath() + "!" + name); } } DocumentSplit split = DocumentSplitFactory.file(fp); split.fileType = fileType; split.innerName = name; splits.add(split); } } return splits; } DocumentSource(TupleFlowParameters parameters); @Override void run(); static List<DocumentSplit> processDirectory(File root, Parameters conf); static List<DocumentSplit> processFile(File fp, Parameters conf); static List<DocumentSplit> processZipFile(File fp, Parameters conf); @Override void setProcessor(Step processor); static void verify(TupleFlowParameters parameters, ErrorStore store); public Processor<DocumentSplit> processor; }
@Test public void testForcedZipFile() throws IOException { File tmp = null; try { tmp = File.createTempFile("zipUtilTest", ".zip"); String fooContents = "foo is the best"; String fooPath = "data/foo.txt"; String trecWebContents = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<DOCHDR>\n" + "http: + "even more text in this part\n" + "</DOCHDR>\n" + "This is some text in a document.\n" + "</DOC>\n"; String trecWebPath = "data/blah/easy.trecweb"; ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(tmp.getAbsolutePath())); ZipUtil.write(zos, fooPath, ByteUtil.fromString(fooContents)); ZipUtil.write(zos, trecWebPath, ByteUtil.fromString(trecWebContents)); zos.close(); ZipFile zipFile = ZipUtil.open(tmp); List<String> entries = ZipUtil.listZipFile(zipFile); assertEquals(2, entries.size()); zipFile.close(); List<DocumentSplit> splits = DocumentSource.processZipFile(tmp, Parameters.parseArray("filetype", "foo")); assertEquals(2, splits.size()); assertEquals("foo", splits.get(0).fileType); assertEquals("foo", splits.get(1).fileType); } finally { if (tmp != null) assertTrue(tmp.delete()); } }
@Test public void testZipFile() throws IOException { File tmp = null; try { tmp = File.createTempFile("zipUtilTest", ".zip"); String fooContents = "foo is the best"; String fooPath = "data/foo.txt"; String barContents = "bar is the best"; String barPath = "data/subdir/ignore/bar.txt"; String trecWebContents = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<DOCHDR>\n" + "http: + "even more text in this part\n" + "</DOCHDR>\n" + "This is some text in a document.\n" + "</DOC>\n"; String trecWebPath = "data/blah/easy.trecweb"; String guessTrecWebPath = "data/blah/guess_trecweb"; String trecTextContents = "<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<DOCHDR>\n" + "http: + "even more text in this part\n" + "</DOCHDR>\n" + "<TEXT>\n" + "This is some text in a document.\n" + "</TEXT>\n" + "</DOC>\n"; String guessTrecTextPath = "data/blah/easy"; String trecTextWithBlankLinesContents = "\n \n<DOC>\n" + "<DOCNO>CACM-0001</DOCNO>\n" + "<DOCHDR>\n" + "http: + "even more text in this part\n" + "</DOCHDR>\n" + "<TEXT>\n" + "This is some text in a document.\n" + "</TEXT>\n" + "</DOC>\n"; String guessTrecTextWithBlankLinesPath = "data/blah/easy2"; String emptyContents = "\n \n\n"; String guessEmptyPath = "data/blah/empty"; ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(tmp.getAbsolutePath())); ZipUtil.write(zos, fooPath, ByteUtil.fromString(fooContents)); ZipUtil.write(zos, barPath, ByteUtil.fromString(barContents)); ZipUtil.write(zos, trecWebPath, ByteUtil.fromString(trecWebContents)); ZipUtil.write(zos, guessTrecWebPath, ByteUtil.fromString(trecWebContents)); ZipUtil.write(zos, guessTrecTextPath, ByteUtil.fromString(trecTextContents)); ZipUtil.write(zos, guessTrecTextWithBlankLinesPath, ByteUtil.fromString(trecTextWithBlankLinesContents)); ZipUtil.write(zos, guessEmptyPath, ByteUtil.fromString(emptyContents)); zos.close(); ZipFile zipFile = ZipUtil.open(tmp); List<String> entries = ZipUtil.listZipFile(zipFile); assertEquals(7, entries.size()); zipFile.close(); List<DocumentSplit> splits = DocumentSource.processZipFile(tmp, Parameters.create()); assertEquals(7, splits.size()); assertEquals("txt", splits.get(0).fileType); assertEquals("txt", splits.get(1).fileType); assertEquals("trecweb", splits.get(2).fileType); assertEquals("trecweb", splits.get(3).fileType); assertEquals("trectext", splits.get(4).fileType); assertEquals("trectext", splits.get(5).fileType); assertEquals(null, splits.get(6).fileType); } finally { if (tmp != null) assertTrue(tmp.delete()); } }