ReferenceExtractor implements Closeable { public String extractAllReferencesString(String text, boolean filterDuplicate, int consolidate, boolean includeRawCitations, List<PatentItem> patents, List<BibDataSet> articles) { try { if (patents == null) { patents = new ArrayList<PatentItem>(); } if (articles == null) { articles = new ArrayList<BibDataSet>(); } if (patentParser == null) { patentParser = new PatentRefParser(); } ArrayList<String> patentBlocks = new ArrayList<String>(); text = text.replace("\n", " ").replace("\t", " "); Language lang = languageUtilities.runLanguageId(text, 500); List<String> tokenizations = analyzer.tokenize(text, lang); int offset = 0; if (tokenizations.size() == 0) { return null; } List<OffsetPosition> journalPositions = null; List<OffsetPosition> abbrevJournalPositions = null; List<OffsetPosition> conferencePositions = null; List<OffsetPosition> publisherPositions = null; { journalPositions = lexicon.tokenPositionsJournalNames(text); abbrevJournalPositions = lexicon.tokenPositionsAbbrevJournalNames(text); conferencePositions = lexicon.tokenPositionsConferenceNames(text); publisherPositions = lexicon.tokenPositionsPublisherNames(text); } boolean isJournalToken = false; boolean isAbbrevJournalToken = false; boolean isConferenceToken = false; boolean isPublisherToken = false; int currentJournalPositions = 0; int currentAbbrevJournalPositions = 0; int currentConferencePositions = 0; int currentPublisherPositions = 0; boolean skipTest = false; int posit = 0; for(String tok : tokenizations) { isJournalToken = false; isAbbrevJournalToken = false; isConferenceToken = false; isPublisherToken = false; skipTest = false; if ( (tok.trim().length() == 0) || (tok.equals(" ")) || (tok.equals("\t")) || (tok.equals("\n")) || (tok.equals("\r")) ) { continue; } if (journalPositions != null) { if (currentJournalPositions == journalPositions.size() - 1) { if (journalPositions.get(currentJournalPositions).end < posit) { skipTest = true; } } if (!skipTest) { for (int i = currentJournalPositions; i < journalPositions.size(); i++) { if ((journalPositions.get(i).start <= posit) && (journalPositions.get(i).end >= posit)) { isJournalToken = true; currentJournalPositions = i; break; } else if (journalPositions.get(i).start > posit) { isJournalToken = false; currentJournalPositions = i; break; } } } } skipTest = false; if (abbrevJournalPositions != null) { if (currentAbbrevJournalPositions == abbrevJournalPositions.size() - 1) { if (abbrevJournalPositions.get(currentAbbrevJournalPositions).end < posit) { skipTest = true; } } if (!skipTest) { for (int i = currentAbbrevJournalPositions; i < abbrevJournalPositions.size(); i++) { if ((abbrevJournalPositions.get(i).start <= posit) && (abbrevJournalPositions.get(i).end >= posit)) { isAbbrevJournalToken = true; currentAbbrevJournalPositions = i; break; } else if (abbrevJournalPositions.get(i).start > posit) { isAbbrevJournalToken = false; currentAbbrevJournalPositions = i; break; } } } } skipTest = false; if (conferencePositions != null) { if (currentConferencePositions == conferencePositions.size() - 1) { if (conferencePositions.get(currentConferencePositions).end < posit) { skipTest = true; } } if (!skipTest) { for (int i = currentConferencePositions; i < conferencePositions.size(); i++) { if ((conferencePositions.get(i).start <= posit) && (conferencePositions.get(i).end >= posit)) { isConferenceToken = true; currentConferencePositions = i; break; } else if (conferencePositions.get(i).start > posit) { isConferenceToken = false; currentConferencePositions = i; break; } } } } skipTest = false; if (publisherPositions != null) { if (currentPublisherPositions == publisherPositions.size() - 1) { if (publisherPositions.get(currentPublisherPositions).end < posit) { skipTest = true; } } if (!skipTest) { for (int i = currentPublisherPositions; i < publisherPositions.size(); i++) { if ((publisherPositions.get(i).start <= posit) && (publisherPositions.get(i).end >= posit)) { isPublisherToken = true; currentPublisherPositions = i; break; } else if (publisherPositions.get(i).start > posit) { isPublisherToken = false; currentPublisherPositions = i; break; } } } } FeaturesVectorReference featureVector = FeaturesVectorReference.addFeaturesPatentReferences(tok, tokenizations.size(), posit, isJournalToken, isAbbrevJournalToken, isConferenceToken, isPublisherToken); patentBlocks.add(featureVector.printVector()); posit++; } patentBlocks.add("\n"); String theResult = null; theResult = taggerAll.label(patentBlocks); StringTokenizer stt = new StringTokenizer(theResult, "\n"); List<String> referencesPatent = new ArrayList<String>(); List<String> referencesNPL = new ArrayList<String>(); List<Integer> offsets_patent = new ArrayList<Integer>(); List<Integer> offsets_NPL = new ArrayList<Integer>(); List<Double> probPatent = new ArrayList<Double>(); List<Double> probNPL = new ArrayList<Double>(); boolean currentPatent = true; String reference = null; double currentProb = 0.0; offset = 0; int currentOffset = 0; int addedOffset = 0; String label = null; String actual = null; int p = 0; while (stt.hasMoreTokens()) { String line = stt.nextToken(); if (line.trim().length() == 0) { continue; } StringTokenizer st2 = new StringTokenizer(line, "\t "); boolean start = true; String separator = ""; label = null; actual = null; while (st2.hasMoreTokens()) { if (start) { actual = st2.nextToken().trim(); start = false; boolean strop = false; while ((!strop) && (p < tokenizations.size())) { String tokOriginal = tokenizations.get(p); addedOffset += tokOriginal.length(); if (tokOriginal.equals(" ")) { separator += tokOriginal; } else if (tokOriginal.equals(actual)) { strop = true; } p++; } } else { label = st2.nextToken().trim(); } } if (label == null) { offset += addedOffset; addedOffset = 0; continue; } double prob = 0.0; int segProb = label.lastIndexOf("/"); if (segProb != -1) { String probString = label.substring(segProb+1, label.length()); try { prob = Double.parseDouble(probString); } catch(Exception e) { LOGGER.debug(probString + " cannot be parsed."); } label = label.substring(0,segProb); } if (actual != null) { if (label.endsWith("<refPatent>")) { if (reference == null) { reference = separator + actual; currentOffset = offset; currentPatent = true; currentProb = prob; } else { if (currentPatent) { if (label.equals("I-<refPatent>")) { referencesPatent.add(reference); offsets_patent.add(currentOffset); probPatent.add(new Double(currentProb)); currentPatent = true; reference = separator + actual; currentOffset = offset; currentProb = prob; } else { reference += separator + actual; if (prob > currentProb) { currentProb = prob; } } } else { referencesNPL.add(reference); offsets_NPL.add(currentOffset); probNPL.add(new Double(currentProb)); currentPatent = true; reference = separator + actual; currentOffset = offset; currentProb = prob; } } } else if (label.endsWith("<refNPL>")) { if (reference == null) { reference = separator + actual; currentOffset = offset; currentPatent = false; currentProb = prob; } else { if (currentPatent) { referencesPatent.add(reference); offsets_patent.add(currentOffset); probPatent.add(new Double(currentProb)); currentPatent = false; reference = separator + actual; currentOffset = offset; currentProb = prob; } else { if (label.equals("I-<refNPL>")) { referencesNPL.add(reference); offsets_NPL.add(currentOffset); probNPL.add(new Double(currentProb)); currentPatent = false; reference = separator + actual; currentOffset = offset; currentProb = prob; } else { reference += separator + actual; if (prob > currentProb) { currentProb = prob; } } } } } else if (label.equals("<other>")) { if (reference != null) { if (currentPatent) { referencesPatent.add(reference); offsets_patent.add(currentOffset); probPatent.add(new Double(currentProb)); } else { referencesNPL.add(reference); offsets_NPL.add(currentOffset); probNPL.add(new Double(currentProb)); } currentPatent = false; } reference = null; currentProb = 0.0; } } offset += addedOffset; addedOffset = 0; } int j = 0; for (String ref : referencesPatent) { patentParser.setRawRefText(ref); patentParser.setRawRefTextOffset(offsets_patent.get(j).intValue()); List<PatentItem> patents0 = patentParser.processRawRefText(); for (PatentItem pat : patents0) { pat.setContext(ref); pat.setConf(probPatent.get(j).doubleValue()); patents.add(pat); } j++; } List<String> numberListe = new ArrayList<String>(); if (filterDuplicate) { List<PatentItem> toRemove = new ArrayList<PatentItem>(); for (PatentItem pat : patents) { if (!numberListe.contains(pat.getNumberEpoDoc())) { numberListe.add(pat.getNumberEpoDoc()); } else { toRemove.add(pat); } } for (PatentItem pat : toRemove) { patents.remove(pat); } } if (articles != null) { int k = 0; for (String ref : referencesNPL) { BiblioItem result = parsers.getCitationParser().processing(ref, consolidate); BibDataSet bds = new BibDataSet(); result.setReference(ref); bds.setResBib(result); bds.setRawBib(ref); bds.addOffset(offsets_NPL.get(k).intValue()); articles.add(bds); k++; } } } catch (Exception e) { throw new GrobidException("An exception occured while running Grobid.", e); } int nbs = 0; if (patents != null) { nbs = patents.size(); } if (articles != null) nbs += articles.size(); String resultTEI = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<TEI xml:space=\"preserve\" xmlns=\"http: "xmlns:xlink=\"http: String divID = KeyGen.getKey().substring(0,7); resultTEI += "<teiHeader />\n"; resultTEI += "<text>\n"; resultTEI += "<div id=\"_"+ divID +"\">\n"; resultTEI += TextUtilities.HTMLEncode(text); resultTEI += "</div>\n"; resultTEI += "<div type=\"references\">\n"; if ( (patents != null) || (articles != null) ) { resultTEI += "<listBibl>\n"; } if (patents != null) { for(PatentItem patentCitation : patents) { resultTEI += patentCitation.toTEI(true, divID) + "\n"; } } if (articles != null) { for(BibDataSet articleCitation : articles) { resultTEI += articleCitation.toTEI(includeRawCitations) + "\n"; } } if ( (patents != null) || (articles != null) ) { resultTEI += "</listBibl>\n"; } resultTEI += "</div>\n"; resultTEI += "</text>\n"; resultTEI += "</TEI>"; return resultTEI; } ReferenceExtractor();  ReferenceExtractor(EngineParsers parsers); void setDocumentPath(String dirName); String extractAllReferencesOPS(boolean filterDuplicate,
                                       int consolidate,
                                       boolean includeRawCitations,
                                       List<PatentItem> patents,
                                       List<BibDataSet> articles); String extractPatentReferencesXMLFile(String pathXML,
                                              boolean filterDuplicate,
                                              int consolidate,
                                              boolean includeRawCitations,
                                              List<PatentItem> patents); String extractAllReferencesXMLFile(String pathXML,
                                           boolean filterDuplicate,
                                           int consolidate,
                                           boolean includeRawCitations,
                                           List<PatentItem> patents,
                                           List<BibDataSet> articles); String extractAllReferencesPDFFile(String inputFile,
                                           boolean filterDuplicate,
                                           int consolidate,
                                           boolean includeRawCitations,
                                           List<PatentItem> patents,
                                           List<BibDataSet> articles); String annotateAllReferencesPDFFile(String inputFile,
                                           boolean filterDuplicate,
                                           int consolidate,
                                           boolean includeRawCitations,
                                           List<PatentItem> patents,
                                           List<BibDataSet> articles); String extractAllReferencesString(String text,
                                          boolean filterDuplicate,
                                          int consolidate,
                                          boolean includeRawCitations,
                                          List<PatentItem> patents,
                                          List<BibDataSet> articles); String annotateAllReferences(Document doc,
										List<LayoutToken> tokenizations,
                                        boolean filterDuplicate,
                                        int consolidate,
                                        boolean includeRawCitations,
                                        List<PatentItem> patents,
                                        List<BibDataSet> articles); String references2TEI2(); String reference2TEI(int i); String references2BibTeX(); String references2TEI(); String reference2BibTeX(int i); void generateTrainingData(String documentPath, String newTrainingPath); boolean getDocOPS(String number); void generateXMLReport(File file,
                                  ArrayList<PatentItem> patents,
                                  ArrayList<BibDataSet> articles); @Override void close(); public boolean debug; public Lexicon lexicon; public String currentPatentNumber; public OPSService ops; public ArrayList<org.grobid.core.data.BibDataSet> resBib; }
@Test public void extractAllReferencesStringNull() { ReferenceExtractor extractor = new ReferenceExtractor(); String res = extractor .extractAllReferencesString( "Economic Development Quarterly November 2011 25: 353-365, first published on August 25, 2011.", false, 0, false, null, null); }
@Test public void extractAllReferencesStringArticles() { ReferenceExtractor extractor = new ReferenceExtractor(); List<PatentItem> patents = new ArrayList<PatentItem>(); List<BibDataSet> articles = new ArrayList<BibDataSet>(); String toExtract = "Some other description includes ref. US 2011/0155847 A1 in aerodynamic" + " and applied physics. " + "This patent, ref. US 7930197 says data mining of personal data is patented. " + "That article refers to Economic Development Quarterly November 2011 25: 353-365, first" + " published on August 25, 2011."; GrobidTimer timer = new GrobidTimer(true); extractor.extractAllReferencesString(toExtract, false, 0, false, patents, articles); timer.stop("STOP"); System.out.println(timer.getElapsedTimeFromStartFormated("STOP")); LOGGER.info("BibDataSet: " + articles.toString()); assertEquals(2, patents.size()); assertEquals(1, articles.size()); LOGGER.info(articles.get(0).getOffsets().toString()); }
@Test public void extractAllReferencesStringArticles2() { ReferenceExtractor extractor = new ReferenceExtractor(); List<PatentItem> patents = new ArrayList<PatentItem>(); List<BibDataSet> articles = new ArrayList<BibDataSet>(); extractor .extractAllReferencesString( "That article It refers to Economic Development Quarterly November 2011 25: 353-365," + " first published on August 25, 2011.", false, 0, false, patents, articles); LOGGER.info("BibDataSet: " + articles.toString()); assertEquals(0, patents.size()); assertEquals(1, articles.size()); List<Integer> offsets = articles.get(0).getOffsets(); int startOffset = -1; int endOffset = -1; String rawBib = articles.get(0).getRawBib(); if (!offsets.isEmpty() && offsets.get(0) != null) { startOffset = offsets.get(0).intValue(); StringTokenizer stt = new StringTokenizer(rawBib, delimiters, true); int count2 = 0; int charCpt = 0; String token2; while (stt.hasMoreTokens()) { token2 = stt.nextToken(); if (token2.trim().length() != 0) { count2++; } charCpt += token2.length(); System.err.println(token2 + " count=" + count2 + " charCpt=" + charCpt); } endOffset = startOffset + count2; System.out.println("RawBib=" + rawBib); System.out.println("Start=" + startOffset + " offset=" + count2 + " end=" + endOffset); } LOGGER.info(articles.get(0).getOffsets().toString()); }
@Test public void jaProcessing() { String text_jp = "すなわち、相対的な頻度で、エポキシドをベースとする液体接着剤及び接着結合剤が、" + "例えばＷＯ９８／２１２８７Ａ１。これらの主な使用分野は、硬質装置のみならず適度に柔軟な装置における縁部の結合である。" + "硬化は、熱により又はＵＶ照射により行われる。"; System.out.println(text_jp); ReferenceExtractor extractor = new ReferenceExtractor(); List<PatentItem> patents = new ArrayList<PatentItem>(); extractor.extractAllReferencesString(text_jp, false, 0, false, patents, null); LOGGER.info("PatentItem: " + patents.toString()); assertEquals(1, patents.size()); assertEquals("21287", patents.get(0).getNumberEpoDoc()); }
@Test public void krProcessing() { String text_kr = "미국의 애플사의 미국 출원 2012/012710."; System.out.println(text_kr); ReferenceExtractor extractor = new ReferenceExtractor(); List<PatentItem> patents = new ArrayList<PatentItem>(); extractor.extractAllReferencesString(text_kr, false, 0, false, patents, null); LOGGER.info("PatentItem: " + patents.toString()); assertEquals(1, patents.size()); assertEquals("2012012710", patents.get(0).getNumberEpoDoc()); }
@Test public void zhProcessing() { String text_zh = "在本申请的申请人于2008年8月26日提交的申请号为US2008/001534的PCT国际申请中，" + "揭示了一种等截面三角形定向棱镜圆形反光板及由其制成的圆板灯。该圆板灯包括：等截面三角形微棱镜圆形导光板；" + "围绕导光板的散热框，该散热框与导光板之间形成间隙而构成环形灯槽；以及嵌装于环形灯槽内的环形灯组件，" + "该环形灯组件由多个发光二极管(LED)贴片、电阻和线路板构成。该申请的全部内容，通过引用结合于此。"; System.out.println(text_zh); ReferenceExtractor extractor = new ReferenceExtractor(); List<PatentItem> patents = new ArrayList<PatentItem>(); extractor.extractAllReferencesString(text_zh, false, 0, false, patents, null); LOGGER.info("PatentItem: " + patents.toString()); assertEquals(1, patents.size()); assertEquals("2008001534", patents.get(0).getNumberEpoDoc()); }