@Test public void testProcessData() throws IOException { NutchDocument nutchDocumentIn = new NutchDocument(); CrawlDatum crawDatum = new CrawlDatum(); Parse parse = mock(Parse.class); Configuration configuration = mock(Configuration.class); ParseData parseData = new ParseData(); parseData.setParseMeta(metadata1); when(parse.getData()).thenReturn(parseData); csvDataFlow.init(configuration, entryList); csvDataFlow.processData(nutchDocumentIn, parse, null, crawDatum, null); parseData.setParseMeta(metadata2); csvDataFlow.processData(nutchDocumentIn, parse, null, crawDatum, null); parseData.setParseMeta(metadata3); csvDataFlow.processData(nutchDocumentIn, parse, null, crawDatum, null); File[] files = tempDir.listFiles(); InputStream createdCsvInputStream = new FileInputStream(files[0]); byte[] createdCsvData = new byte[createdCsvInputStream.available()]; createdCsvInputStream.read(createdCsvData); String createdCsv = new String(createdCsvData); assertEquals("Something went wrong with the creation of the csv file!", 1, files.length); assertEquals("CSV is not as predicted!", preCreatedCsv, createdCsv); csvDataFlow.destroy(); }
@Override public void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { Metadata metadata = parse.getData().getParseMeta(); try { initWriters(); for(Entry entry : entryList) { CSVPrint csvPrint = nameCsvPrintMap.get(entry.getParameterMap().get(NAME)); String[] fieldValues = new String[entry.getFieldList().size()]; List<Field> fieldList = entry.getFieldList(); for (int i = 0; i < fieldList.size(); i++) { fieldValues[i] = FilterUtils.getNullSafe(metadata.get(fieldList.get(i).getName()), ""); } csvPrint.println(fieldValues); } } catch (IOException e) { e.printStackTrace(); } }
CsvDataFlow implements DataFlow { @Override public void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { Metadata metadata = parse.getData().getParseMeta(); try { initWriters(); for(Entry entry : entryList) { CSVPrint csvPrint = nameCsvPrintMap.get(entry.getParameterMap().get(NAME)); String[] fieldValues = new String[entry.getFieldList().size()]; List<Field> fieldList = entry.getFieldList(); for (int i = 0; i < fieldList.size(); i++) { fieldValues[i] = FilterUtils.getNullSafe(metadata.get(fieldList.get(i).getName()), ""); } csvPrint.println(fieldValues); } } catch (IOException e) { e.printStackTrace(); } } }
CsvDataFlow implements DataFlow { @Override public void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { Metadata metadata = parse.getData().getParseMeta(); try { initWriters(); for(Entry entry : entryList) { CSVPrint csvPrint = nameCsvPrintMap.get(entry.getParameterMap().get(NAME)); String[] fieldValues = new String[entry.getFieldList().size()]; List<Field> fieldList = entry.getFieldList(); for (int i = 0; i < fieldList.size(); i++) { fieldValues[i] = FilterUtils.getNullSafe(metadata.get(fieldList.get(i).getName()), ""); } csvPrint.println(fieldValues); } } catch (IOException e) { e.printStackTrace(); } } CsvDataFlow(); }
CsvDataFlow implements DataFlow { @Override public void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { Metadata metadata = parse.getData().getParseMeta(); try { initWriters(); for(Entry entry : entryList) { CSVPrint csvPrint = nameCsvPrintMap.get(entry.getParameterMap().get(NAME)); String[] fieldValues = new String[entry.getFieldList().size()]; List<Field> fieldList = entry.getFieldList(); for (int i = 0; i < fieldList.size(); i++) { fieldValues[i] = FilterUtils.getNullSafe(metadata.get(fieldList.get(i).getName()), ""); } csvPrint.println(fieldValues); } } catch (IOException e) { e.printStackTrace(); } } CsvDataFlow(); @Override void init(Configuration configuration, List<Entry> entryList); @Override void destroy(); @Override void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks); }
CsvDataFlow implements DataFlow { @Override public void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) { Metadata metadata = parse.getData().getParseMeta(); try { initWriters(); for(Entry entry : entryList) { CSVPrint csvPrint = nameCsvPrintMap.get(entry.getParameterMap().get(NAME)); String[] fieldValues = new String[entry.getFieldList().size()]; List<Field> fieldList = entry.getFieldList(); for (int i = 0; i < fieldList.size(); i++) { fieldValues[i] = FilterUtils.getNullSafe(metadata.get(fieldList.get(i).getName()), ""); } csvPrint.println(fieldValues); } } catch (IOException e) { e.printStackTrace(); } } CsvDataFlow(); @Override void init(Configuration configuration, List<Entry> entryList); @Override void destroy(); @Override void processData(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks); static final QName DESTINATION; static final QName NAME; static final QName SEPERATOR; static final QName QUOTE_CHARACTER; static final QName APPEND_TIMESTAMP; static final QName ADD_FIELD_NAMES_TO_FIRST_LINE; static final QName STYLE; }