RobotsServiceImpl implements RobotsService { @Override public boolean isAllowed(@Nonnull String crawlerAgentString, @Nonnull URI resourceUri) { checkNotNull(crawlerAgentString, "crawlerAgentString is null"); checkNotNull(resourceUri, "resourceUri is null"); log.debug("Resolving robots URL for: {}", resourceUri); final URI robotsUri = utilities.getRobotsURIForResource(resourceUri); log.debug("Resolved robots URI to: {}", robotsUri); final Robots robots; try { robots = loader.load(robotsUri); } catch (Exception e) { log.debug("Download failure {}", e.getMessage()); return allow(resourceUri); } if (robots.getGroups().isEmpty()) { log.debug("No agent groups found", resourceUri); return allow(resourceUri); } final Optional<Group> group = matcherUtils.getMostSpecificMatchingGroup(robots.getGroups(), crawlerAgentString); if (group.isPresent()) { if(log.isDebugEnabled()) { log.debug("Matched user-agent group: {}", matcherUtils.getMostSpecificMatch(group.get(), crawlerAgentString).get().getValue()); } } else { log.debug("No user-agent group matched"); return allow(resourceUri); } final Optional<PathDirective> bestMatch = matcherUtils.getMostSpecificMatch( group.get().getDirectives(PathDirective.class), utilities.getResourceLocalComponents(resourceUri)); if (!bestMatch.isPresent()) { log.debug("No matching path directive"); return allow(resourceUri); } else { final PathDirective directive = bestMatch.get(); log.debug("Matched path directive {}:{}", directive.getField(), directive.getValue()); return directive.isAllowed() ? allow(resourceUri) : disallow(resourceUri); } } RobotsServiceImpl(@Nonnull RobotsLoader loader, @Nonnull RobotsUtilities utilities, MatcherUtils matcherUtils); @Override boolean isAllowed(@Nonnull String crawlerAgentString, @Nonnull URI resourceUri); @Override void close();  }
@Test(expected = NullPointerException.class) public void givenNullUri_whenIsAllowed_thenThrowsNPE() { instance.isAllowed(CRAWLER_AGENT, null); }
@Test(expected = NullPointerException.class) public void givenNullAgent_whenIsAllowed_thenThrowsNPE() { instance.isAllowed(null, RESOURCE_URI); }
@Test public void givenExampleUri_whenIsAllowed_thenReturnsTrue() { boolean result = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(result, is(true)); }
@Test public void givenExampleUri_whenIsAllowed_thenSourceFactoryIsInvoked() { instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); verify(utilities).getRobotsURIForResource(RESOURCE_URI); }
@Test(expected = IllegalArgumentException.class) public void givenMalformedResourceURI_whenIsAllowed_thenThrowsIAE() { when(utilities.getRobotsURIForResource(RESOURCE_URI)).thenThrow(new IllegalArgumentException()); instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); }
@Test public void givenLoaderThrowsException_whenIsAllowed_thenReturnsTrue() throws Exception { when(loader.load(any(URI.class))).thenThrow(Exception.class); boolean allowed = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(allowed, equalTo(true)); }
@Test public void givenLoaderProducesSingleNonMatchingGroup_whenIsAllowed_thenReturnsTrue() throws Exception { when(loader.load(any(URI.class))).thenReturn(SINGLE_GROUP_ROBOTS); when(matcherUtils.getMostSpecificMatchingGroup(anyCollection(), anyString())).thenReturn(Optional.absent()); boolean allowed = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(allowed, equalTo(true)); }
@Test public void givenLoaderProducesMatchingGroup_butNotMatchingPath_whenIsAllowed_thenReturnsTrue() throws Exception { when(loader.load(any(URI.class))).thenReturn(SINGLE_GROUP_ROBOTS); when(matcherUtils.getMostSpecificMatchingGroup(anyCollection(), anyString())).thenReturn(Optional.of(GROUP)); when(matcherUtils.getMostSpecificMatch(anyCollection(), anyString())).thenReturn(Optional.absent()); boolean allowed = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(allowed, equalTo(true)); }
@Test public void givenLoaderProducesMatchingGroup_andSingleMatchingPath_andFieldIsAllow_whenIsAllowed_thenReturnsTrue() throws Exception { when(loader.load(any(URI.class))).thenReturn(SINGLE_GROUP_ROBOTS); when(matcherUtils.getMostSpecificMatchingGroup(anyCollection(), anyString())).thenReturn(Optional.of(GROUP)); when(matcherUtils.getMostSpecificMatch(anyCollection(), anyString())).thenReturn(Optional.of(ALLOW_PATH_DIRECTIVE)); boolean allowed = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(allowed, equalTo(true)); }
@Test public void givenLoaderProducesMatchingGroup_andSingleMatchingPath_andFieldIsDisallow_whenIsAllowed_thenReturnsTrue() throws Exception { when(loader.load(any(URI.class))).thenReturn(SINGLE_GROUP_ROBOTS); when(matcherUtils.getMostSpecificMatchingGroup(anyCollection(), anyString())).thenReturn(Optional.of(GROUP)); when(matcherUtils.getMostSpecificMatch(anyCollection(), anyString())).thenReturn(Optional.of(DISALLOW_PATH_DIRECTIVE)); boolean allowed = instance.isAllowed(CRAWLER_AGENT, RESOURCE_URI); assertThat(allowed, equalTo(false)); }