SeedUriDomainnameQueueAssignmentPolicy extends HostnameQueueAssignmentPolicy { public String getClassKey(CrawlURI cauri) { log.debug("Finding classKey for cauri: " + cauri); String key = null; if (treatDnsLikeHttp || !isDns(cauri)) { key = getKeyFromSeed(cauri); } if (key == null) { key = getKeyFromUriHostname(cauri); } if (key == null) { key = DEFAULT_CLASS_KEY; } return key; }  boolean isTreatDnsLikeHttp(); void setTreatDnsLikeHttp(boolean treatDnsLikeHttp); String getClassKey(CrawlURI cauri);  }
@Test public void getClassKey() throws Exception { SeedUriDomainnameQueueAssignmentPolicy policy = new SeedUriDomainnameQueueAssignmentPolicy(); String url1 = "http: String url2 = "http: UURI uuri1 = UURIFactory.getInstance(url1); UURI uuri2 = UURIFactory.getInstance(url2); CrawlURI curi1 = new CrawlURI(uuri1); curi1.setSeed(true); curi1.setSourceTag(url1); CrawlURI curi2 = curi1.createCrawlURI(uuri2, LinkContext.EMBED_MISC, Hop.EMBED); assertEquals(policy.getClassKey(curi1), policy.getClassKey(curi2)); }
@Test public void testGetKeyNoSchema() throws Exception { SeedUriDomainnameQueueAssignmentPolicy policy = new SeedUriDomainnameQueueAssignmentPolicy(); String url1 = "http: UURI uuri1 = UURIFactory.getInstance(url1); CrawlURI curi1 = new CrawlURI(uuri1); curi1.setSeed(true); curi1.setSourceTag("www.ssdown.dk"); assertEquals("ssdown.dk", policy.getClassKey(curi1)); }
@Test public void getClassKeyTestChain() throws Exception { SeedUriDomainnameQueueAssignmentPolicy policy = new SeedUriDomainnameQueueAssignmentPolicy(); String url1 = "http: UURI uuri1 = UURIFactory.getInstance(url1); UURI uuri2 = UURIFactory.getInstance("http: UURI uuri3 = UURIFactory.getInstance("http: UURI uuri4 = UURIFactory.getInstance("http: UURI uuri5 = UURIFactory.getInstance("http: UURI uuri6 = UURIFactory.getInstance("http: UURI uuri7 = UURIFactory.getInstance("http: CrawlURI curi1 = new CrawlURI(uuri1); curi1.setSeed(true); curi1.setSourceTag(url1); CrawlURI curi2 = curi1.createCrawlURI(uuri2, null, Hop.NAVLINK); CrawlURI curi3 = curi2.createCrawlURI(uuri3, null, Hop.REFER ); CrawlURI curi4 = curi3.createCrawlURI(uuri4, null, Hop.NAVLINK); CrawlURI curi5 = curi4.createCrawlURI(uuri5, null, Hop.EMBED); CrawlURI curi6 = curi5.createCrawlURI(uuri6, null, Hop.PREREQ); CrawlURI curi7 = curi6.createCrawlURI(uuri7, null, Hop.INFERRED); assertEquals(policy.getClassKey(curi7), policy.getClassKey(curi1)); }