@Test public void testMapOnValidWebcrawlResources() throws Exception { List<DocumentText> webcrawlResources = loadResources("wos_document_text.json"); for (DocumentText text : webcrawlResources) { ingester.map(new AvroKey<DocumentText>(text), null, context); } verify(context, times(3)).write(resultCaptor.capture(), any()); DocumentText value = resultCaptor.getAllValues().get(0).datum(); assertNotNull(value); assertEquals("id-1", value.getId().toString()); assertEquals("\"This work was partially supported by the 7th EC Framework Programme " + "Project \"\"NEWTBVac\"\" Grant #241745.\"", value.getText()); value = resultCaptor.getAllValues().get(1).datum(); assertNotNull(value); assertEquals("id-2", value.getId().toString()); assertEquals("This work was supported by the Archimedes Center for Modeling, " + "Analysis and Computation (ACMAC) (project FP7-REGPOT-2009-1).", value.getText()); value = resultCaptor.getAllValues().get(2).datum(); assertNotNull(value); assertEquals("id-3", value.getId().toString()); assertEquals("This work was supported by the Archimedes Center for Modeling, " + "Analysis and Computation (ACMAC) (project FP7-REGPOT-2009-1).\n" + "\"This work is part of the research program of the \"\"Stichting voor " + "Fundamenteel Onderzoek der Materie (FOM)\"\", which is financially " + "supported by the \"\"Nederlandse organisatie voor Wetenschappelijk " + "Onderzock (NWO)\"\". D.P. thanks the European Commission for support " + "with a Marie Curic grant (Project No. TERASPIN 039223).\"", value.getText()); }
@Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } } }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }  }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }   }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }    }