@Test public void testListAttachments() throws Exception { doc.putContentField("attachment_links", Arrays.asList(mockUrl, mockUrl, mockUrl)); stage.process(doc); Assert.assertTrue(doc.hasContentField("links_content")); Assert.assertTrue(doc.hasContentField("links2_content")); Assert.assertTrue(doc.hasContentField("links3_content")); }
@Override public void process(LocalDocument doc) throws Exception { Map<String, Object> urls = FieldHelper.getFieldMatchingPattern(doc.getContentMap(), urlFieldPattern); UriParser uriParser = new UriParser(); DocumentParserHelper documentParserHelper = new DocumentParserHelper(addMetaData, addLanguage); InputStreamParser inputStreamParser = new InputStreamParser(parser); for (String field : urls.keySet()) { Iterator<URL> it = uriParser.getUrlsFromObject(urls.get(field)).iterator(); for (int i = 1; it.hasNext(); i++) { String num = (i > 1) ? "" + i : ""; URL url = it.next(); URLConnection connection = createConnection(url); final InputStream inputStream = connection.getInputStream(); try { String prefix = field + num + "_"; ParsedData parsedData = inputStreamParser.parse(inputStream); documentParserHelper.addParsedDataToDocument(parsedData, doc, prefix); } finally { inputStream.close(); } } } }
SimpleFetchingTikaStage extends AbstractProcessStage { @Override public void process(LocalDocument doc) throws Exception { Map<String, Object> urls = FieldHelper.getFieldMatchingPattern(doc.getContentMap(), urlFieldPattern); UriParser uriParser = new UriParser(); DocumentParserHelper documentParserHelper = new DocumentParserHelper(addMetaData, addLanguage); InputStreamParser inputStreamParser = new InputStreamParser(parser); for (String field : urls.keySet()) { Iterator<URL> it = uriParser.getUrlsFromObject(urls.get(field)).iterator(); for (int i = 1; it.hasNext(); i++) { String num = (i > 1) ? "" + i : ""; URL url = it.next(); URLConnection connection = createConnection(url); final InputStream inputStream = connection.getInputStream(); try { String prefix = field + num + "_"; ParsedData parsedData = inputStreamParser.parse(inputStream); documentParserHelper.addParsedDataToDocument(parsedData, doc, prefix); } finally { inputStream.close(); } } } } }
SimpleFetchingTikaStage extends AbstractProcessStage { @Override public void process(LocalDocument doc) throws Exception { Map<String, Object> urls = FieldHelper.getFieldMatchingPattern(doc.getContentMap(), urlFieldPattern); UriParser uriParser = new UriParser(); DocumentParserHelper documentParserHelper = new DocumentParserHelper(addMetaData, addLanguage); InputStreamParser inputStreamParser = new InputStreamParser(parser); for (String field : urls.keySet()) { Iterator<URL> it = uriParser.getUrlsFromObject(urls.get(field)).iterator(); for (int i = 1; it.hasNext(); i++) { String num = (i > 1) ? "" + i : ""; URL url = it.next(); URLConnection connection = createConnection(url); final InputStream inputStream = connection.getInputStream(); try { String prefix = field + num + "_"; ParsedData parsedData = inputStreamParser.parse(inputStream); documentParserHelper.addParsedDataToDocument(parsedData, doc, prefix); } finally { inputStream.close(); } } } }  }
SimpleFetchingTikaStage extends AbstractProcessStage { @Override public void process(LocalDocument doc) throws Exception { Map<String, Object> urls = FieldHelper.getFieldMatchingPattern(doc.getContentMap(), urlFieldPattern); UriParser uriParser = new UriParser(); DocumentParserHelper documentParserHelper = new DocumentParserHelper(addMetaData, addLanguage); InputStreamParser inputStreamParser = new InputStreamParser(parser); for (String field : urls.keySet()) { Iterator<URL> it = uriParser.getUrlsFromObject(urls.get(field)).iterator(); for (int i = 1; it.hasNext(); i++) { String num = (i > 1) ? "" + i : ""; URL url = it.next(); URLConnection connection = createConnection(url); final InputStream inputStream = connection.getInputStream(); try { String prefix = field + num + "_"; ParsedData parsedData = inputStreamParser.parse(inputStream); documentParserHelper.addParsedDataToDocument(parsedData, doc, prefix); } finally { inputStream.close(); } } } }  @Override void process(LocalDocument doc); @Override void init(); }
SimpleFetchingTikaStage extends AbstractProcessStage { @Override public void process(LocalDocument doc) throws Exception { Map<String, Object> urls = FieldHelper.getFieldMatchingPattern(doc.getContentMap(), urlFieldPattern); UriParser uriParser = new UriParser(); DocumentParserHelper documentParserHelper = new DocumentParserHelper(addMetaData, addLanguage); InputStreamParser inputStreamParser = new InputStreamParser(parser); for (String field : urls.keySet()) { Iterator<URL> it = uriParser.getUrlsFromObject(urls.get(field)).iterator(); for (int i = 1; it.hasNext(); i++) { String num = (i > 1) ? "" + i : ""; URL url = it.next(); URLConnection connection = createConnection(url); final InputStream inputStream = connection.getInputStream(); try { String prefix = field + num + "_"; ParsedData parsedData = inputStreamParser.parse(inputStream); documentParserHelper.addParsedDataToDocument(parsedData, doc, prefix); } finally { inputStream.close(); } } } }  @Override void process(LocalDocument doc); @Override void init(); @Parameter(required = true, description = "The field name pattern that should be matched where " +
            "urls will be found. First group plus \"_\" will be used as field prefix. Example:" +
            " \"attachment_(.*)\" will match for example attachment_a and will use \"a_\" as prefix")
    public String urlFieldPattern; @Parameter(name = "addMetaData", description = "Add the metadata to the document or not. Defaults to true")
    public boolean addMetaData; @Parameter(description = "Set to true, will also do language detection and add the field 'prefix_language' according to the prefix rules. Defaults to true")
    public boolean addLanguage; @Parameter(description = "Username for basic authentication.")
    public String username; @Parameter(description = "Password for basic authentication.")
    public String password; }