AnnotationsGenerateFn extends AppFunction { public void run (Parameters p, PrintStream output) throws Exception { String annotationTypeString = null; String[] annotationTypes = null; String index = null; if (p.isString ("outputFile")) { boolean append = p.get ("appendFile", false); PrintStream out = new PrintStream (new BufferedOutputStream ( new FileOutputStream (p.getString ("outputFile"), append)), true, "UTF-8"); } if (!p.containsKey ("annotationTypes")) { output.println ("generate-annotations requires an \'annotationTypes\' parameter."); output.println (this.getHelpString ()); return; } else { annotationTypesHS = new HashSet<> (); annotationTypeString = p.getString ("annotationTypes"); annotationTypes = annotationTypeString.split ("[ ,]+"); for (String type : annotationTypes) { if (type.toLowerCase().equals ("all")) { annotationTypesHS.add ("person"); annotationTypesHS.add ("location"); annotationTypesHS.add ("organization"); break; } if (!type.equals ("person") && !type.equals ("location") && !type.equals ("organization")) { output.printf ("Unknown annotation type ^%s^. Must be person, organization or location. Ignoring type. %n", type); continue; } annotationTypesHS.add (type.toLowerCase ()); } if (annotationTypesHS.size () == 0) { output.println ("No recognized annotation types defined. Quitting."); return; } } initClassifier (output); if (p.containsKey ("indexPath")) { String indexPath = null; try { if (!p.containsKey ("eidList") && !p.containsKey ("iidList")) { output.println ("generate-annotations requires an \'iidList\' and/or \'eidList\' parameter when using an existing index."); output.println (this.getHelpString ()); return; } indexPath = (new File (p.getString ("indexPath"))).getAbsolutePath (); File indexFile = new File (indexPath); assert (indexFile.isDirectory ()); Map<Long, String> iid2eidTM = new TreeMap<>(); long docCount = 0L; if (p.containsKey ("iidList")) { String idsStr = p.getString ("iidList"); String[] iidsList = idsStr.split ("[ ,]+"); int len = iidsList.length; if (len > 0) { DiskNameReader dnr = new DiskNameReader (indexPath + File.separator + "names"); if (dnr.getManifest ().getBoolean ("emptyIndexFile")) { output.println ("Empty Names Index File. Quitting."); return; } docCount = dnr.getManifest().get ("keyCount", 0); for (String id : iidsList) { long iid = Long.parseLong (id); String eid = dnr.getDocumentName (iid); if (eid == null) { output.println ("IID " + id + " does not exist in the index. Skipping."); continue; } if (!iid2eidTM.containsKey (iid)) { iid2eidTM.put (iid, eid); } } dnr.close(); } } if (p.containsKey ("eidList")) { String idsStr = p.getString ("eidList"); String[] eidsList = idsStr.split ("[ ,]+"); int len = eidsList.length; if (len > 0) { DiskNameReverseReader dnrr = new DiskNameReverseReader ( indexPath + File.separator + "names.reverse"); for (String str : eidsList) { String eidStr = str.trim (); long iid = dnrr.getDocumentIdentifier (eidStr); if (iid == -1) { output.println ("EID " + eidStr + " does not exist in the index. Skipping."); continue; } if (iid > -1) { if (!iid2eidTM.containsKey (iid)) { iid2eidTM.put (iid, eidStr); } } } dnrr.close(); } } Parameters docParams = Parameters.create (); docParams.set ("index", indexPath); docParams.set ("tokenize", false); docParams.set ("text", true); docParams.set ("metadata", false); DocumentComponents dc = null; Retrieval r = null; dc = new DocumentComponents (docParams); r = RetrievalFactory.instance (indexPath, Parameters.create ()); assert r.getAvailableParts ().containsKey ("corpus") : "Index does not contain a corpus part."; long lastDocID = -1; Set tmSet = iid2eidTM.entrySet (); Iterator it = tmSet.iterator (); while (it.hasNext ()) { Map.Entry me = (Map.Entry)it.next (); long iid = ((Long)me.getKey ()).longValue (); String eid = (String)me.getValue (); Document document = r.getDocument (eid, dc); if (document == null) { output.printf ("Document [%d] %s does not exist %n", iid, eid); continue; } else { String docName = document.name; String docText = document.text; if (docText == null) { output.printf ("Document [%d] %s has no text in index ^%s^. %n", iid, eid, indexPath); continue; } else { output.printf ("Doc [%d] %s %n", iid, docName); extractEntities (indexPath, docName, docText, output); } } } } catch (Exception ex) { output.println ("Exception extracting annotations from index ^" + indexPath + "^"); output.println ("Exception: " + ex.toString ()); throw new RuntimeException (ex); } } else { String fileListString = p.getString ("inputFiles"); String[] fileList = fileListString.split ("[ ,]+"); for (String fn : fileList) { File f = new File (fn.trim ()); if (! f.exists ()) { output.println ("File ^" + fn + "^ does not exist."); continue; } List<DocumentSplit>splitList = null; if (f.isDirectory ()) { splitList = DocumentSource.processDirectory (f, Parameters.create ()); } else { splitList = DocumentSource.processFile (f, Parameters.create ()); } for (DocumentSplit ds : splitList) { DocumentStreamParser parser = DocumentStreamParser.create (ds, Parameters.create ()); long fileDocCount = 0L; Document doc = null; while ( (doc = parser.nextDocument ()) != null ) { fileDocCount++; doc.filePath = ds.fileName; doc.fileId = ds.fileId; doc.totalFileCount = ds.totalFileCount; TagTokenizer tt = new TagTokenizer (); tt.tokenize (doc); if (doc != null) { String name = doc.name; String content = doc.text; extractEntities (doc.filePath, name, content, output); } else { output.println ("Doc ^" + doc.filePath + "^ has no content"); } } } } } }  static void main(String[] args); @Override String getName(); @Override String getHelpString(); void run(Parameters p, PrintStream output); static final Logger logger; }
@Test public void testSingleIID () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); App.run (new String[]{ "annotations-generate", "--indexPath=" + indexFilePath, "--iidList=1,", "--annotationTypes=person,location,organization" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); assertTrue ("Output should contain " + outputLines.length, outputLines.length == 2); String[] expectedLines = { "Doc [1] tt-0002", indexFilePath + " tt-0002 organization university_of_missouri-rolla 16 18 121 149" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim (); String outputL = outputLines[i].trim (); assertTrue ("Output line [" + i + "] doesn't match expected", expectedL.equals (outputL)); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testSingleEID () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); App.run (new String[]{ "annotations-generate", "--indexPath=" + indexFilePath, "--eidList=tt-0235,", "--annotationTypes=all" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); assertTrue ("Output should contain " + outputLines.length, outputLines.length == 3); String[] expectedLines = { "Doc [2] tt-0235 ", indexFilePath + " tt-0235 organization chrysler 0 1 14 22", indexFilePath + " tt-0235 organization chrysler_motors_corp. 5 7 60 81" }; int i = 0; for (String line : expectedLines) { String outputL = outputLines[i].trim(); String expectedL = line.trim(); assertTrue ("Output line [" + i + "] doesn't match expected", expectedL.equals (outputL)); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testUndefIID () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); App.run (new String[]{ "annotations-generate", "--indexPath=" + indexFilePath, "--iidList=999,", "--annotationTypes=person,location,organization" }, printStream); String outputLine = byteArrayStream.toString ().trim (); String expectedLine = "IID 999 does not exist in the index. Skipping."; assertTrue ("Output line [" + expectedLine + "] doesn't match expected", expectedLine.equals (outputLine)); byteArrayStream.close(); printStream.close(); }
@Test public void testUndefEID () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); App.run (new String[]{ "annotations-generate", "--indexPath=" + indexFilePath, "--eidList=XYZ", "--annotationTypes=ALL" }, printStream); String outputLine = byteArrayStream.toString ().trim (); String expectedLine = "EID XYZ does not exist in the index. Skipping."; assertTrue ("Output line [" + outputLine + "] doesn't match expected", expectedLine.equals (outputLine)); byteArrayStream.close(); printStream.close(); }
@Test public void testIIDsandEIDs () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); App.run (new String[]{ "annotations-generate", "--indexPath=" + indexFilePath, "--iidList=1,2", "--eidList=tt-0235,tt-1", "--annotationTypes=organization,location"}, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split(System.lineSeparator()); assertTrue ("Output should contain " + outputLines.length, outputLines.length == 8); String[] expectedLines = { "Doc [0] tt-1 ", indexFilePath + " tt-1 organization bureau_of_alcohol_tobacco 7 10 93 118", indexFilePath + " tt-1 location maryland 19 20 178 186", "Doc [1] tt-0002 ", indexFilePath + " tt-0002 organization university_of_missouri-rolla 16 18 121 149", "Doc [2] tt-0235 ", indexFilePath + " tt-0235 organization chrysler 0 1 14 22", indexFilePath + " tt-0235 organization chrysler_motors_corp. 5 7 60 81" }; int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testTrecTextOrganization () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File trecTextFile = FileUtility.createTemporary (); trecTextFile.deleteOnExit(); String trecTextFilePath = trecTextFile.getAbsolutePath (); StreamUtil.copyStringToFile (trecTextDocument, trecTextFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + trecTextFilePath, "--annotationTypes=organization" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { trecTextFilePath + " tt-1 organization bureau_of_alcohol_tobacco_and_firearms 8 13 88 126", trecTextFilePath + " tt-1 organization state_police 31 32 252 264", }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 2); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testTrecTextPerson () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File trecTextFile = FileUtility.createTemporary (); trecTextFile.deleteOnExit(); String trecTextFilePath = trecTextFile.getAbsolutePath (); StreamUtil.copyStringToFile (trecTextDocument, trecTextFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + trecTextFilePath, "--annotationTypes=person" }, printStream); String output = byteArrayStream.toString (); String outputLine = output.trim (); String expectedLine = trecTextFilePath + " tt-1 person john_smith 35 36 286 296"; assertEquals (expectedLine, outputLine); byteArrayStream.close(); printStream.close(); }
@Test public void testTrecTextLocation () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File trecTextFile = FileUtility.createTemporary (); trecTextFile.deleteOnExit(); String trecTextFilePath = trecTextFile.getAbsolutePath (); StreamUtil.copyStringToFile (trecTextDocument, trecTextFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + trecTextFilePath, "--annotationTypes=location" }, printStream); String output = byteArrayStream.toString (); String outputLine = output.trim (); String expectedLine = trecTextFilePath + " tt-1 location maryland 21 22 175 183"; assertEquals (expectedLine, outputLine); byteArrayStream.close(); printStream.close(); }
@Test public void testTrecTextALL () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File trecTextFile = FileUtility.createTemporary (); trecTextFile.deleteOnExit(); String trecTextFilePath = trecTextFile.getAbsolutePath (); StreamUtil.copyStringToFile (trecTextDocument, trecTextFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + trecTextFilePath, "--annotationTypes=All" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { trecTextFilePath + " tt-1 organization bureau_of_alcohol_tobacco_and_firearms 8 13 88 126", trecTextFilePath + " tt-1 location maryland 21 22 175 183", trecTextFilePath + " tt-1 organization state_police 31 32 252 264", trecTextFilePath + " tt-1 person john_smith 35 36 286 296" }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 4); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testTrecWeb () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File trecWebFile = FileUtility.createTemporary (); trecWebFile.deleteOnExit(); String trecWebFilePath = trecWebFile.getAbsolutePath (); StreamUtil.copyStringToFile (trecWebDocument, trecWebFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + trecWebFilePath, "--annotationTypes=ALL" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { trecWebFilePath + " tw-04 person tom_jones 0 1 0 9", trecWebFilePath + " tw-04 organization society_of_manufacturing_engineers 4 7 17 51", trecWebFilePath + " tw-04 person smith 10 11 61 66", trecWebFilePath + " tw-04 location paducah_kentucky 24 25 152 168" }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 4); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testTextFile () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File textFile = File.createTempFile (tmpDirStr + File.separator +"tempTextFile", ".txt"); textFile.deleteOnExit(); String textFilePath = textFile.getAbsolutePath (); StreamUtil.copyStringToFile (textDocument, textFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + textFilePath, "--annotationTypes=ALL" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { textFilePath + " " + textFilePath + " person george_shultz 6 7 26 39", textFilePath + " " + textFilePath + " location boston_massachusetts 12 13 56 76", textFilePath + " " + textFilePath + " location washington_d.c. 21 22 101 116", textFilePath + " " + textFilePath + " person ronald_reagan 24 25 122 135", textFilePath + " " + textFilePath + " organization republican_party 35 36 175 191" }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 5); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testXMLFile () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File xmlFile = File.createTempFile (tmpDirStr + File.separator +"tempXmlFile", ".xml"); xmlFile.deleteOnExit(); String xmlFilePath = xmlFile.getAbsolutePath (); StreamUtil.copyStringToFile (xmlDocument, xmlFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + xmlFilePath, "--annotationTypes=ALL" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { xmlFilePath + " " + xmlFilePath + " person george_washington_carver 3 5 52 76", xmlFilePath + " " + xmlFilePath + " location georgia 8 9 105 112", xmlFilePath + " " + xmlFilePath + " location missouri 10 11 135 143", xmlFilePath + " " + xmlFilePath + " location iowa 12 13 169 173", xmlFilePath + " " + xmlFilePath + " location usa 14 15 175 178", xmlFilePath + " " + xmlFilePath + " organization naacp 19 20 229 234" }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 6); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testTrecTextMultiDocs () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File multiDocFile = FileUtility.createTemporary (); multiDocFile.deleteOnExit(); String multiDocFilePath = multiDocFile.getAbsolutePath (); StreamUtil.copyStringToFile (multiDocument, multiDocFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + multiDocFilePath, "--annotationTypes=ALL" }, printStream); String output = byteArrayStream.toString (); String[] outputLines = output.split (System.lineSeparator ()); String[] expectedLines = { multiDocFilePath + " tt-1 organization bureau_of_alcohol_tobacco_and_firearms 8 13 88 126", multiDocFilePath + " tt-1 location maryland 21 22 174 182", multiDocFilePath + " tt-0002 organization university_of_missouri-rolla 16 18 115 143", multiDocFilePath + " tt-0235 organization chrysler 0 1 7 15", multiDocFilePath + " tt-0235 organization chrysler_motors_corp. 5 7 54 75" }; assertTrue ("Output should contain " + expectedLines.length, outputLines.length == 5); int i = 0; for (String line : expectedLines) { String expectedL = line.trim(); String outputL = outputLines[i].trim(); assertEquals (expectedL, outputL); i++; } byteArrayStream.close(); printStream.close(); }
@Test public void testNoEntitiesDoc () throws Exception { ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream (); PrintStream printStream = new PrintStream (byteArrayStream); File nuthinDocFile = FileUtility.createTemporary (); nuthinDocFile.deleteOnExit(); String nuthinDocFilePath = nuthinDocFile.getAbsolutePath (); StreamUtil.copyStringToFile (nuthinDocument, nuthinDocFile); App.run (new String[]{ "annotations-generate", "--inputFiles=" + nuthinDocFilePath, "--annotationTypes=ALL" }, printStream); String outputLine = byteArrayStream.toString ().trim (); String expectedLine = nuthinDocFilePath + " " + "[tw-no-entities] No extractions this document."; assertEquals (expectedLine, outputLine); byteArrayStream.close(); printStream.close(); }