@Test public void testMap() throws Exception { Configuration conf = new Configuration(); conf.set(NAMED_OUTPUT_META, "meta"); conf.set(NAMED_OUTPUT_FAULT, "fault"); doReturn(conf).when(context).getConfiguration(); mapper.setup(context); String id = "id"; DocumentText.Builder docTextBuilder = DocumentText.newBuilder(); docTextBuilder.setId(id); docTextBuilder.setText(ClassPathResourceProvider.getResourceContent(XML_FILE)); mapper.map(new AvroKey<>(docTextBuilder.build()), null, context); verify(context, never()).write(any(), any()); verify(multipleOutputs, times(1)).write(mosKeyCaptor.capture(), mosValueCaptor.capture()); assertEquals(conf.get(NAMED_OUTPUT_META), mosKeyCaptor.getValue()); ExtractedDocumentMetadata docMeta = (ExtractedDocumentMetadata) mosValueCaptor.getValue().datum(); assertNotNull(docMeta); assertEquals(id, docMeta.getId()); }
@Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText nlm = key.datum(); String documentId = nlm.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (!StringUtils.isBlank(nlm.getText())) { final ExtractedDocumentMetadata.Builder output = ExtractedDocumentMetadata.newBuilder(); output.setId(nlm.getId()); try { String pmcXml = nlm.getText().toString(); output.setText(extractText(pmcXml, oaiNamespace)); extractMetadata(pmcXml, output); mos.write(namedOutputMeta, new AvroKey<ExtractedDocumentMetadata>(output.build())); } catch (Exception e) { handleException(nlm, e, output); } } }
MetadataImporter extends Mapper<AvroKey<DocumentText>, NullWritable, NullWritable, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText nlm = key.datum(); String documentId = nlm.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (!StringUtils.isBlank(nlm.getText())) { final ExtractedDocumentMetadata.Builder output = ExtractedDocumentMetadata.newBuilder(); output.setId(nlm.getId()); try { String pmcXml = nlm.getText().toString(); output.setText(extractText(pmcXml, oaiNamespace)); extractMetadata(pmcXml, output); mos.write(namedOutputMeta, new AvroKey<ExtractedDocumentMetadata>(output.build())); } catch (Exception e) { handleException(nlm, e, output); } } } }
MetadataImporter extends Mapper<AvroKey<DocumentText>, NullWritable, NullWritable, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText nlm = key.datum(); String documentId = nlm.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (!StringUtils.isBlank(nlm.getText())) { final ExtractedDocumentMetadata.Builder output = ExtractedDocumentMetadata.newBuilder(); output.setId(nlm.getId()); try { String pmcXml = nlm.getText().toString(); output.setText(extractText(pmcXml, oaiNamespace)); extractMetadata(pmcXml, output); mos.write(namedOutputMeta, new AvroKey<ExtractedDocumentMetadata>(output.build())); } catch (Exception e) { handleException(nlm, e, output); } } }  }
MetadataImporter extends Mapper<AvroKey<DocumentText>, NullWritable, NullWritable, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText nlm = key.datum(); String documentId = nlm.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (!StringUtils.isBlank(nlm.getText())) { final ExtractedDocumentMetadata.Builder output = ExtractedDocumentMetadata.newBuilder(); output.setId(nlm.getId()); try { String pmcXml = nlm.getText().toString(); output.setText(extractText(pmcXml, oaiNamespace)); extractMetadata(pmcXml, output); mos.write(namedOutputMeta, new AvroKey<ExtractedDocumentMetadata>(output.build())); } catch (Exception e) { handleException(nlm, e, output); } } }  @Override void cleanup(Context context); }
MetadataImporter extends Mapper<AvroKey<DocumentText>, NullWritable, NullWritable, NullWritable> { @Override protected void map(AvroKey<DocumentText> key, NullWritable value, Context context) throws IOException, InterruptedException { DocumentText nlm = key.datum(); String documentId = nlm.getId().toString(); if (excludedIds.contains(documentId)) { log.info("skipping processing for excluded id " + documentId); return; } if (!StringUtils.isBlank(nlm.getText())) { final ExtractedDocumentMetadata.Builder output = ExtractedDocumentMetadata.newBuilder(); output.setId(nlm.getId()); try { String pmcXml = nlm.getText().toString(); output.setText(extractText(pmcXml, oaiNamespace)); extractMetadata(pmcXml, output); mos.write(namedOutputMeta, new AvroKey<ExtractedDocumentMetadata>(output.build())); } catch (Exception e) { handleException(nlm, e, output); } } }  @Override void cleanup(Context context); static final String NAMED_OUTPUT_META; static final String NAMED_OUTPUT_FAULT; static final String EXCLUDED_IDS; static final String FAULT_TEXT; static final String PARAM_INGEST_METADATA_OAI_NAMESPACE; }