@Test public void testUrlExistsAndNeedsToBeUpdated() throws Exception { String url = "http: Entry entry = createEntry(url, 2880); when(robotsRepository.findByHost(anyString())).thenReturn(entry); when(redisUrlRepository.exists(anyString())).thenReturn(false); consumerService.receiveUrl(url); String baseHost = DomainUtils.base(new URI(url)); String robotUrl = String.format("%s/robots.txt", baseHost); verify(downloaderService, times(1)).download(eq(baseHost), eq(robotUrl)); }
public void receiveUrl(String url) { if (!UrlUtils.isValid(url)) { log.error("Url {} is not valid", url); return; } log.info("Received url {}, prepared to fetch robots.txt file for it", url); try { URI uri = new URI(url); String baseHost = DomainUtils.base(uri); log.debug("Validated url to base host: {}", baseHost); checkHost(baseHost); } catch (URISyntaxException e) { log.error("Can't create URI using url {}", url, e); } }
IncomingUrlService { public void receiveUrl(String url) { if (!UrlUtils.isValid(url)) { log.error("Url {} is not valid", url); return; } log.info("Received url {}, prepared to fetch robots.txt file for it", url); try { URI uri = new URI(url); String baseHost = DomainUtils.base(uri); log.debug("Validated url to base host: {}", baseHost); checkHost(baseHost); } catch (URISyntaxException e) { log.error("Can't create URI using url {}", url, e); } } }
IncomingUrlService { public void receiveUrl(String url) { if (!UrlUtils.isValid(url)) { log.error("Url {} is not valid", url); return; } log.info("Received url {}, prepared to fetch robots.txt file for it", url); try { URI uri = new URI(url); String baseHost = DomainUtils.base(uri); log.debug("Validated url to base host: {}", baseHost); checkHost(baseHost); } catch (URISyntaxException e) { log.error("Can't create URI using url {}", url, e); } }  }
IncomingUrlService { public void receiveUrl(String url) { if (!UrlUtils.isValid(url)) { log.error("Url {} is not valid", url); return; } log.info("Received url {}, prepared to fetch robots.txt file for it", url); try { URI uri = new URI(url); String baseHost = DomainUtils.base(uri); log.debug("Validated url to base host: {}", baseHost); checkHost(baseHost); } catch (URISyntaxException e) { log.error("Can't create URI using url {}", url, e); } }  void receiveUrl(String url); }
IncomingUrlService { public void receiveUrl(String url) { if (!UrlUtils.isValid(url)) { log.error("Url {} is not valid", url); return; } log.info("Received url {}, prepared to fetch robots.txt file for it", url); try { URI uri = new URI(url); String baseHost = DomainUtils.base(uri); log.debug("Validated url to base host: {}", baseHost); checkHost(baseHost); } catch (URISyntaxException e) { log.error("Can't create URI using url {}", url, e); } }  void receiveUrl(String url);  }