RDFAnalyzer extends AbstractAnalyzer { @Override public Iterator<byte[]> analyze(CrawleableUri curi, File data, Sink sink) { FileInputStream fin = null; try { LOGGER.info("Starting the RDF Analyzer for URI: " + curi.getUri().toString()); Lang lang = null; Object httpMimeTypeObject = curi.getData(Constants.URI_HTTP_MIME_TYPE_KEY); String contentType = null; if ((httpMimeTypeObject != null) && (!"text/plain".equals(httpMimeTypeObject.toString()))) { contentType = httpMimeTypeObject.toString(); } StreamRDF filtered = new FilterSinkRDF(curi, sink, collector,tripleEncoder); if (contentType != null) { lang = RDFLanguages.contentTypeToLang(contentType); try { RDFDataMgr.parse(filtered, data.getAbsolutePath(), lang); } catch (Exception e) { LOGGER.warn("Could not parse file as " + lang.getName()); } } else { LOGGER.info("Content Type is null"); BufferedInputStream dataStream = new BufferedInputStream(Files.newInputStream(data.toPath())); Collection<Lang> langs = serializationDetector.detect(dataStream); LOGGER.info("Detected languages: {}", langs); for (Lang l : langs) { try { if (dataStream != null) { RDFDataMgr.parse(filtered, dataStream, l); } else { RDFDataMgr.parse(filtered, data.getAbsolutePath(), l); } break; } catch (Exception e) { LOGGER.warn("Could not parse file as " + l.getName()); failedParseAttempts += 1; } finally { if (dataStream != null) { dataStream.close(); dataStream = null; } } } } ActivityUtil.addStep(curi, getClass()); return collector.getUris(curi); } catch (Exception e) { LOGGER.error("Exception while analyzing. Aborting. ", e); ActivityUtil.addStep(curi, getClass(), e.getMessage()); return null; } finally { IOUtils.closeQuietly(fin); } } RDFAnalyzer(UriCollector collector); @Override Iterator<byte[]> analyze(CrawleableUri curi, File data, Sink sink); boolean isElegible(CrawleableUri curi, File data);  }
@Test public void test() throws URISyntaxException, IOException { Serializer serializer = new GzipJavaUriSerializer(); SimpleUriCollector collector = new SimpleUriCollector(serializer); Sink sink = new InMemorySink(); Analyzer analyzer = new RDFAnalyzer(collector); CrawleableUri curi = new CrawleableUri(new URI("http: File dataFile = File.createTempFile("analyzer-test-", ""); try (InputStream in = this.getClass().getClassLoader().getResourceAsStream(resourceName); OutputStream out = new BufferedOutputStream(new FileOutputStream(dataFile))) { IOUtils.copy(in, out); } sink.openSinkForUri(curi); collector.openSinkForUri(curi); startTime = stopwatch.runtime(TimeUnit.MILLISECONDS); analyzer.analyze(curi, dataFile, sink); endTime = stopwatch.runtime(TimeUnit.MILLISECONDS); Assert.assertEquals("Number of triples in " + resourceName, expectedNumberOfTriples, collector.getSize()); Assert.assertTrue("Failed parse attempts for " + resourceName + ": " + ((RDFAnalyzer)analyzer).failedParseAttempts + " <= 2", ((RDFAnalyzer)analyzer).failedParseAttempts <= 2); sink.closeSinkForUri(curi); collector.closeSinkForUri(curi); }