@Test public void testLinksExtraction() throws IOException { final URL SAMPLE_RESOURCE = Thread.currentThread().getContextClassLoader().getResource("links_extract.html"); assertNotNull("The sample file should be resolved", SAMPLE_RESOURCE); final File SAMPLE = new File(SAMPLE_RESOURCE.getFile()); final URL CONF_RESOURCE = Thread.currentThread().getContextClassLoader().getResource("links_extract.conf"); assertNotNull("The config file should be resolved", CONF_RESOURCE); final File CONF = new File(CONF_RESOURCE.getFile()); Config config = ConfigFactory.parseFile(CONF); HTMLAnalyser ha = new HTMLAnalyser(config); Map<String, Object> core = new HashMap<String, Object>(); core.put("subject-uri", "NotPresent"); core.put("ip-address", "192.168.1.10"); core.put("creation-date", "Invalid"); core.put("content-type", "text/html"); core.put("length", Long.toString(SAMPLE.length())); core.put("version", "InvalidVersion"); core.put("absolute-offset", "0"); core.put("origin", ""); ArchiveRecordHeader header = new ARCRecordMetaData("invalid", core); SolrRecord solr = SolrRecordFactory.createFactory(null).createRecord(); InputStream in = new BufferedInputStream(new FileInputStream(SAMPLE), (int) SAMPLE.length()); in.mark((int) SAMPLE.length()); ha.analyse("source", header, in, solr); assertEquals("The number of links should be correct", 4, solr.getField(SolrFields.SOLR_LINKS).getValueCount()); assertEquals("The number of hosts should be correct", 1, solr.getField(SolrFields.SOLR_LINKS_HOSTS).getValueCount()); String host = (String) solr.getField(SolrFields.SOLR_LINKS_HOSTS) .getFirstValue(); assertEquals("The host should be formatted correctly", "example.org", host); String domain = (String) solr.getField(SolrFields.SOLR_LINKS_DOMAINS) .getFirstValue(); assertEquals("The domain should be formatted correctly", "example.org", domain); String suffix = (String) solr.getField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES).getFirstValue(); assertEquals("The suffix should be formatted correctly", "org", suffix); String[] linksDomainsSurts = new String[]{"(org,", "(org,example,"}; assertArrayEquals("The SURT domains should be correct", linksDomainsSurts, solr.getField(SolrFields.SOLR_LINKS_HOSTS_SURTS).getValues() .toArray()); }
@Override public void analyse(String source, ArchiveRecordHeader header, InputStream tikainput, SolrRecord solr) { final long start = System.nanoTime(); Metadata metadata = new Metadata(); Set<String> hosts = new HashSet<String>(); Set<String> suffixes = new HashSet<String>(); Set<String> domains = new HashSet<String>(); metadata.set( Metadata.RESOURCE_NAME_KEY, Normalisation.sanitiseWARCHeaderValue(header.getUrl()) ); ParseRunner parser = new ParseRunner( hfp, tikainput, metadata, solr ); try { TimeLimiter.run(parser, 30000L, false); } catch( Exception e ) { log.error( "WritableSolrRecord.extract(): " + e.getMessage() ); solr.addParseException("when parsing as HTML", e); } Instrument.timeRel("HTMLAnalyzer.analyze#total", "HTMLAnalyzer.analyze#parser", start); if (extractImageLinks){ String[] imageLinks = metadata.getValues( HtmlFeatureParser.IMAGE_LINKS ); if (imageLinks != null){ for( String link : imageLinks ) { String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS_IMAGES, urlNorm); } } } String[] links_list = metadata.getValues( HtmlFeatureParser.LINK_LIST ); if( links_list != null ) { String lhost, ldomain, lsuffix; for( String link : links_list ) { lhost = LinkExtractor.extractHost( link ); if( !lhost.equals( LinkExtractor.MALFORMED_HOST ) ) { hosts.add(lhost); } lsuffix = LinkExtractor.extractPublicSuffix( link ); if( lsuffix != null ) { suffixes.add(lsuffix); } ldomain = LinkExtractor.extractPrivateSuffix( link ); if( ldomain != null ) { domains.add(ldomain); } if( this.extractLinks ){ String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS, urlNorm ); } } if( this.extractLinkHosts ) { Set<String> cHostSet = new HashSet<String>(); for (String host: hosts) { String cHost = host; if (WARCIndexer.CANONICALISE_HOST) { try { cHost = Normalisation.canonicaliseHost(host); } catch (URIException e) { log.error("Failed to canonicalise host: '" + host + "': " + e); cHost = host; } } cHostSet.add(cHost); } Set<String> uniqueHostLevels = new LinkedHashSet<String>(); for (String host : cHostSet) { solr.addField(SolrFields.SOLR_LINKS_HOSTS, host); uniqueHostLevels.addAll(LinkExtractor.allLevels(host)); } for(String hostLevel : uniqueHostLevels) { solr.addField(SolrFields.SOLR_LINKS_HOSTS_SURTS, SURT.toSURT(hostLevel)); } } if( this.extractLinkDomains ) { for (String domain: domains) { solr.addField( SolrFields.SOLR_LINKS_DOMAINS, domain ); } } for (String suffix: suffixes) { solr.addField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES, suffix ); } } if( this.extractElementsUsed ) { String[] de = metadata.getValues( HtmlFeatureParser.DISTINCT_ELEMENTS ); if( de != null ) { for( String e : de ) { solr.addField( SolrFields.ELEMENTS_USED, e ); } } } for( String lurl : metadata.getValues( Metadata.LICENSE_URL ) ) { solr.addField( SolrFields.LICENSE_URL, lurl ); } Instrument.timeRel("WARCPayloadAnalyzers.analyze#total", "HTMLAnalyzer.analyze#total", start); }
HTMLAnalyser extends AbstractPayloadAnalyser { @Override public void analyse(String source, ArchiveRecordHeader header, InputStream tikainput, SolrRecord solr) { final long start = System.nanoTime(); Metadata metadata = new Metadata(); Set<String> hosts = new HashSet<String>(); Set<String> suffixes = new HashSet<String>(); Set<String> domains = new HashSet<String>(); metadata.set( Metadata.RESOURCE_NAME_KEY, Normalisation.sanitiseWARCHeaderValue(header.getUrl()) ); ParseRunner parser = new ParseRunner( hfp, tikainput, metadata, solr ); try { TimeLimiter.run(parser, 30000L, false); } catch( Exception e ) { log.error( "WritableSolrRecord.extract(): " + e.getMessage() ); solr.addParseException("when parsing as HTML", e); } Instrument.timeRel("HTMLAnalyzer.analyze#total", "HTMLAnalyzer.analyze#parser", start); if (extractImageLinks){ String[] imageLinks = metadata.getValues( HtmlFeatureParser.IMAGE_LINKS ); if (imageLinks != null){ for( String link : imageLinks ) { String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS_IMAGES, urlNorm); } } } String[] links_list = metadata.getValues( HtmlFeatureParser.LINK_LIST ); if( links_list != null ) { String lhost, ldomain, lsuffix; for( String link : links_list ) { lhost = LinkExtractor.extractHost( link ); if( !lhost.equals( LinkExtractor.MALFORMED_HOST ) ) { hosts.add(lhost); } lsuffix = LinkExtractor.extractPublicSuffix( link ); if( lsuffix != null ) { suffixes.add(lsuffix); } ldomain = LinkExtractor.extractPrivateSuffix( link ); if( ldomain != null ) { domains.add(ldomain); } if( this.extractLinks ){ String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS, urlNorm ); } } if( this.extractLinkHosts ) { Set<String> cHostSet = new HashSet<String>(); for (String host: hosts) { String cHost = host; if (WARCIndexer.CANONICALISE_HOST) { try { cHost = Normalisation.canonicaliseHost(host); } catch (URIException e) { log.error("Failed to canonicalise host: '" + host + "': " + e); cHost = host; } } cHostSet.add(cHost); } Set<String> uniqueHostLevels = new LinkedHashSet<String>(); for (String host : cHostSet) { solr.addField(SolrFields.SOLR_LINKS_HOSTS, host); uniqueHostLevels.addAll(LinkExtractor.allLevels(host)); } for(String hostLevel : uniqueHostLevels) { solr.addField(SolrFields.SOLR_LINKS_HOSTS_SURTS, SURT.toSURT(hostLevel)); } } if( this.extractLinkDomains ) { for (String domain: domains) { solr.addField( SolrFields.SOLR_LINKS_DOMAINS, domain ); } } for (String suffix: suffixes) { solr.addField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES, suffix ); } } if( this.extractElementsUsed ) { String[] de = metadata.getValues( HtmlFeatureParser.DISTINCT_ELEMENTS ); if( de != null ) { for( String e : de ) { solr.addField( SolrFields.ELEMENTS_USED, e ); } } } for( String lurl : metadata.getValues( Metadata.LICENSE_URL ) ) { solr.addField( SolrFields.LICENSE_URL, lurl ); } Instrument.timeRel("WARCPayloadAnalyzers.analyze#total", "HTMLAnalyzer.analyze#total", start); } }
HTMLAnalyser extends AbstractPayloadAnalyser { @Override public void analyse(String source, ArchiveRecordHeader header, InputStream tikainput, SolrRecord solr) { final long start = System.nanoTime(); Metadata metadata = new Metadata(); Set<String> hosts = new HashSet<String>(); Set<String> suffixes = new HashSet<String>(); Set<String> domains = new HashSet<String>(); metadata.set( Metadata.RESOURCE_NAME_KEY, Normalisation.sanitiseWARCHeaderValue(header.getUrl()) ); ParseRunner parser = new ParseRunner( hfp, tikainput, metadata, solr ); try { TimeLimiter.run(parser, 30000L, false); } catch( Exception e ) { log.error( "WritableSolrRecord.extract(): " + e.getMessage() ); solr.addParseException("when parsing as HTML", e); } Instrument.timeRel("HTMLAnalyzer.analyze#total", "HTMLAnalyzer.analyze#parser", start); if (extractImageLinks){ String[] imageLinks = metadata.getValues( HtmlFeatureParser.IMAGE_LINKS ); if (imageLinks != null){ for( String link : imageLinks ) { String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS_IMAGES, urlNorm); } } } String[] links_list = metadata.getValues( HtmlFeatureParser.LINK_LIST ); if( links_list != null ) { String lhost, ldomain, lsuffix; for( String link : links_list ) { lhost = LinkExtractor.extractHost( link ); if( !lhost.equals( LinkExtractor.MALFORMED_HOST ) ) { hosts.add(lhost); } lsuffix = LinkExtractor.extractPublicSuffix( link ); if( lsuffix != null ) { suffixes.add(lsuffix); } ldomain = LinkExtractor.extractPrivateSuffix( link ); if( ldomain != null ) { domains.add(ldomain); } if( this.extractLinks ){ String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS, urlNorm ); } } if( this.extractLinkHosts ) { Set<String> cHostSet = new HashSet<String>(); for (String host: hosts) { String cHost = host; if (WARCIndexer.CANONICALISE_HOST) { try { cHost = Normalisation.canonicaliseHost(host); } catch (URIException e) { log.error("Failed to canonicalise host: '" + host + "': " + e); cHost = host; } } cHostSet.add(cHost); } Set<String> uniqueHostLevels = new LinkedHashSet<String>(); for (String host : cHostSet) { solr.addField(SolrFields.SOLR_LINKS_HOSTS, host); uniqueHostLevels.addAll(LinkExtractor.allLevels(host)); } for(String hostLevel : uniqueHostLevels) { solr.addField(SolrFields.SOLR_LINKS_HOSTS_SURTS, SURT.toSURT(hostLevel)); } } if( this.extractLinkDomains ) { for (String domain: domains) { solr.addField( SolrFields.SOLR_LINKS_DOMAINS, domain ); } } for (String suffix: suffixes) { solr.addField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES, suffix ); } } if( this.extractElementsUsed ) { String[] de = metadata.getValues( HtmlFeatureParser.DISTINCT_ELEMENTS ); if( de != null ) { for( String e : de ) { solr.addField( SolrFields.ELEMENTS_USED, e ); } } } for( String lurl : metadata.getValues( Metadata.LICENSE_URL ) ) { solr.addField( SolrFields.LICENSE_URL, lurl ); } Instrument.timeRel("WARCPayloadAnalyzers.analyze#total", "HTMLAnalyzer.analyze#total", start); } HTMLAnalyser();  HTMLAnalyser( Config conf ); }
HTMLAnalyser extends AbstractPayloadAnalyser { @Override public void analyse(String source, ArchiveRecordHeader header, InputStream tikainput, SolrRecord solr) { final long start = System.nanoTime(); Metadata metadata = new Metadata(); Set<String> hosts = new HashSet<String>(); Set<String> suffixes = new HashSet<String>(); Set<String> domains = new HashSet<String>(); metadata.set( Metadata.RESOURCE_NAME_KEY, Normalisation.sanitiseWARCHeaderValue(header.getUrl()) ); ParseRunner parser = new ParseRunner( hfp, tikainput, metadata, solr ); try { TimeLimiter.run(parser, 30000L, false); } catch( Exception e ) { log.error( "WritableSolrRecord.extract(): " + e.getMessage() ); solr.addParseException("when parsing as HTML", e); } Instrument.timeRel("HTMLAnalyzer.analyze#total", "HTMLAnalyzer.analyze#parser", start); if (extractImageLinks){ String[] imageLinks = metadata.getValues( HtmlFeatureParser.IMAGE_LINKS ); if (imageLinks != null){ for( String link : imageLinks ) { String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS_IMAGES, urlNorm); } } } String[] links_list = metadata.getValues( HtmlFeatureParser.LINK_LIST ); if( links_list != null ) { String lhost, ldomain, lsuffix; for( String link : links_list ) { lhost = LinkExtractor.extractHost( link ); if( !lhost.equals( LinkExtractor.MALFORMED_HOST ) ) { hosts.add(lhost); } lsuffix = LinkExtractor.extractPublicSuffix( link ); if( lsuffix != null ) { suffixes.add(lsuffix); } ldomain = LinkExtractor.extractPrivateSuffix( link ); if( ldomain != null ) { domains.add(ldomain); } if( this.extractLinks ){ String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS, urlNorm ); } } if( this.extractLinkHosts ) { Set<String> cHostSet = new HashSet<String>(); for (String host: hosts) { String cHost = host; if (WARCIndexer.CANONICALISE_HOST) { try { cHost = Normalisation.canonicaliseHost(host); } catch (URIException e) { log.error("Failed to canonicalise host: '" + host + "': " + e); cHost = host; } } cHostSet.add(cHost); } Set<String> uniqueHostLevels = new LinkedHashSet<String>(); for (String host : cHostSet) { solr.addField(SolrFields.SOLR_LINKS_HOSTS, host); uniqueHostLevels.addAll(LinkExtractor.allLevels(host)); } for(String hostLevel : uniqueHostLevels) { solr.addField(SolrFields.SOLR_LINKS_HOSTS_SURTS, SURT.toSURT(hostLevel)); } } if( this.extractLinkDomains ) { for (String domain: domains) { solr.addField( SolrFields.SOLR_LINKS_DOMAINS, domain ); } } for (String suffix: suffixes) { solr.addField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES, suffix ); } } if( this.extractElementsUsed ) { String[] de = metadata.getValues( HtmlFeatureParser.DISTINCT_ELEMENTS ); if( de != null ) { for( String e : de ) { solr.addField( SolrFields.ELEMENTS_USED, e ); } } } for( String lurl : metadata.getValues( Metadata.LICENSE_URL ) ) { solr.addField( SolrFields.LICENSE_URL, lurl ); } Instrument.timeRel("WARCPayloadAnalyzers.analyze#total", "HTMLAnalyzer.analyze#total", start); } HTMLAnalyser();  HTMLAnalyser( Config conf ); @Override void configure(Config conf); @Override boolean shouldProcess(String mime); @Override void analyse(String source, ArchiveRecordHeader header,
            InputStream tikainput, SolrRecord solr); }
HTMLAnalyser extends AbstractPayloadAnalyser { @Override public void analyse(String source, ArchiveRecordHeader header, InputStream tikainput, SolrRecord solr) { final long start = System.nanoTime(); Metadata metadata = new Metadata(); Set<String> hosts = new HashSet<String>(); Set<String> suffixes = new HashSet<String>(); Set<String> domains = new HashSet<String>(); metadata.set( Metadata.RESOURCE_NAME_KEY, Normalisation.sanitiseWARCHeaderValue(header.getUrl()) ); ParseRunner parser = new ParseRunner( hfp, tikainput, metadata, solr ); try { TimeLimiter.run(parser, 30000L, false); } catch( Exception e ) { log.error( "WritableSolrRecord.extract(): " + e.getMessage() ); solr.addParseException("when parsing as HTML", e); } Instrument.timeRel("HTMLAnalyzer.analyze#total", "HTMLAnalyzer.analyze#parser", start); if (extractImageLinks){ String[] imageLinks = metadata.getValues( HtmlFeatureParser.IMAGE_LINKS ); if (imageLinks != null){ for( String link : imageLinks ) { String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS_IMAGES, urlNorm); } } } String[] links_list = metadata.getValues( HtmlFeatureParser.LINK_LIST ); if( links_list != null ) { String lhost, ldomain, lsuffix; for( String link : links_list ) { lhost = LinkExtractor.extractHost( link ); if( !lhost.equals( LinkExtractor.MALFORMED_HOST ) ) { hosts.add(lhost); } lsuffix = LinkExtractor.extractPublicSuffix( link ); if( lsuffix != null ) { suffixes.add(lsuffix); } ldomain = LinkExtractor.extractPrivateSuffix( link ); if( ldomain != null ) { domains.add(ldomain); } if( this.extractLinks ){ String urlNorm = normaliseLinks ? Normalisation.canonicaliseURL(link) : link; solr.addField( SolrFields.SOLR_LINKS, urlNorm ); } } if( this.extractLinkHosts ) { Set<String> cHostSet = new HashSet<String>(); for (String host: hosts) { String cHost = host; if (WARCIndexer.CANONICALISE_HOST) { try { cHost = Normalisation.canonicaliseHost(host); } catch (URIException e) { log.error("Failed to canonicalise host: '" + host + "': " + e); cHost = host; } } cHostSet.add(cHost); } Set<String> uniqueHostLevels = new LinkedHashSet<String>(); for (String host : cHostSet) { solr.addField(SolrFields.SOLR_LINKS_HOSTS, host); uniqueHostLevels.addAll(LinkExtractor.allLevels(host)); } for(String hostLevel : uniqueHostLevels) { solr.addField(SolrFields.SOLR_LINKS_HOSTS_SURTS, SURT.toSURT(hostLevel)); } } if( this.extractLinkDomains ) { for (String domain: domains) { solr.addField( SolrFields.SOLR_LINKS_DOMAINS, domain ); } } for (String suffix: suffixes) { solr.addField( SolrFields.SOLR_LINKS_PUBLIC_SUFFIXES, suffix ); } } if( this.extractElementsUsed ) { String[] de = metadata.getValues( HtmlFeatureParser.DISTINCT_ELEMENTS ); if( de != null ) { for( String e : de ) { solr.addField( SolrFields.ELEMENTS_USED, e ); } } } for( String lurl : metadata.getValues( Metadata.LICENSE_URL ) ) { solr.addField( SolrFields.LICENSE_URL, lurl ); } Instrument.timeRel("WARCPayloadAnalyzers.analyze#total", "HTMLAnalyzer.analyze#total", start); } HTMLAnalyser();  HTMLAnalyser( Config conf ); @Override void configure(Config conf); @Override boolean shouldProcess(String mime); @Override void analyse(String source, ArchiveRecordHeader header,
            InputStream tikainput, SolrRecord solr);  }