ExtractReuters { public static List<ReutersDocument> extract(Path reutersDir) throws IOException, ParseException { List<ReutersDocument> docs = new ArrayList<>(); DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm"); for (Path sgmFile : stream) { InputStream inputStream = Files.newInputStream(sgmFile); docs.addAll(extractFile(inputStream, sgmFile.toUri())); } return docs; }  static List<ReutersDocument> extract(Path reutersDir); static List<ReutersDocument> extractFile(InputStream sgmFile, URI uri);  }
@Test public void testExtract() throws Exception { int expectedDocs = 1000; String expectedTitleFirst = "BAHIA COCOA REVIEW"; Date expectedDateFirst = new GregorianCalendar(1987, 1, 26, 15, 1, 1).getTime(); String expectedBodyFirst = "Showers"; String expectedTopicFirst = "cocoa"; ReutersDocument.LEWISSPLIT expectedlLewissplitFirst = ReutersDocument.LEWISSPLIT.TRAIN; ReutersDocument.CGISPLIT expectedCgisplitFirst = ReutersDocument.CGISPLIT.TRAINING_SET; int oldIdFirst = 5544; int newIdFirst = 1; String expectedTitle4 = "NATIONAL AVERAGE PRICES FOR FARMER-OWNED RESERVE"; Date expectedDate4 = new GregorianCalendar(1987, 1, 26, 15, 10, 44).getTime(); String expectedBody4 = "The U.S. Agriculture Department"; Set<String> expectedTopic4 = new HashSet<>( Arrays.asList( new String[] { "grain", "wheat", "corn", "barley", "oat", "sorghum" })); ReutersDocument.LEWISSPLIT expectedlLewissplit4 = ReutersDocument.LEWISSPLIT.TRAIN; ReutersDocument.CGISPLIT expectedCgisplit4 = ReutersDocument.CGISPLIT.TRAINING_SET; int oldId4 = 5548; int newId4 = 5; String expectedTitleLast = "NATIONAL AMUSEMENTS AGAIN UPS VIACOM <VIA> BID"; Date expectedDateLast = new GregorianCalendar(1987, 2, 3, 9, 17, 32).getTime(); String expectedBodyLast = "Viacom International Inc said "; String expectedTopicLast = "acq"; ReutersDocument.LEWISSPLIT expectedlLewissplitLast = ReutersDocument.LEWISSPLIT.TRAIN; ReutersDocument.CGISPLIT expectedCgisplitLast = ReutersDocument.CGISPLIT.TRAINING_SET; int oldIdLast = 16320; int newIdLast = 1000; List<ReutersDocument> docs = ExtractReuters.extract(new File(REUTERS_DIR).toPath()); assertEquals(expectedDocs, docs.size()); ReutersDocument doc0 = docs.get(0); assertEquals(expectedTitleFirst, doc0.getTitle()); assertEquals(expectedDateFirst.toString(), doc0.getDate().toString()); assertTrue(doc0.getTopics().contains(expectedTopicFirst)); assertTrue(doc0.getBody().startsWith(expectedBodyFirst)); Assert.assertEquals(expectedlLewissplitFirst, doc0.getLewissplit()); Assert.assertEquals(expectedCgisplitFirst, doc0.getCgisplit()); Assert.assertEquals(oldIdFirst, doc0.getOldid()); Assert.assertEquals(newIdFirst, doc0.getNewid()); ReutersDocument doc4 = docs.get(4); assertEquals(expectedTitle4, doc4.getTitle()); assertEquals(expectedDate4.toString(), doc4.getDate().toString()); assertEquals(expectedTopic4, doc4.getTopics()); assertTrue(doc0.getBody().startsWith(expectedBodyFirst)); Assert.assertEquals(expectedlLewissplit4, doc4.getLewissplit()); Assert.assertEquals(expectedCgisplit4, doc4.getCgisplit()); Assert.assertEquals(oldId4, doc4.getOldid()); Assert.assertEquals(newId4, doc4.getNewid()); ReutersDocument doc999 = docs.get(999); assertEquals(expectedTitleLast, doc999.getTitle()); assertEquals(expectedDateLast.toString(), doc999.getDate().toString()); assertTrue(doc999.getTopics().contains(expectedTopicLast)); assertTrue(doc999.getBody().startsWith(expectedBodyLast)); Assert.assertEquals(expectedlLewissplitLast, doc999.getLewissplit()); Assert.assertEquals(expectedCgisplitLast, doc999.getCgisplit()); Assert.assertEquals(oldIdLast, doc999.getOldid()); Assert.assertEquals(newIdLast, doc999.getNewid()); }