@Test public void testMapOnInvalidWebcrawlResources() throws Exception { List<DocumentText> webcrawlResources = loadResources("wos_document_text_broken.json"); for (DocumentText text : webcrawlResources) { ingester.map(new AvroKey<DocumentText>(text), null, context); } verify(context, times(0)).write(any(), any()); }
@Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } } }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }  }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }   }
WebcrawlFundingsIngester extends Mapper<AvroKey<DocumentText>, NullWritable, AvroKey<DocumentText>, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText xmlText = key.datum(); if (!StringUtils.isBlank(xmlText.getText())) { try { SAXParserFactory saxFactory = SAXParserFactory.newInstance(); saxFactory.setValidating(false); SAXParser saxParser = saxFactory.newSAXParser(); XMLReader reader = saxParser.getXMLReader(); reader.setFeature("http: reader.setFeature("http: reader.setFeature("http: WebcrawlFundingsHandler xmlHandler = new WebcrawlFundingsHandler(); saxParser.parse(new InputSource(new StringReader(xmlText.getText().toString())), xmlHandler); if (!StringUtils.isBlank(xmlHandler.getFundingText())) { DocumentText.Builder output = DocumentText.newBuilder(); output.setId(xmlText.getId()); output.setText(xmlHandler.getFundingText()); context.write(new AvroKey<DocumentText>(output.build()), NullWritable.get()); } } catch (Exception e) { log.error("Fundings text extraction failed for id " + xmlText.getId() + " and text: " + xmlText.getText(), e); } } }    }