RuleMatcher { public boolean canCrawl(String agent, String path, String rules) { if (path == null || path.isEmpty()) path = "/"; if ("/robots.txt".equals(path)) return true; Content content = gson.fromJson(rules, type); List<Group> groups = content.getGroups(); if (groups.isEmpty()) return true; List<String> userAgents = groups.stream() .flatMap(group -> group.getUserAgents().stream().map(String::toLowerCase)) .collect(Collectors.toList()); if (userAgents.isEmpty()) return true; if (userAgents.parallelStream().noneMatch(ALL::equals) && ALL.equals(agent)) return true; agent = agent.toLowerCase(); if (noneMatch(agent, userAgents) && noneMatch(ALL, userAgents) && !ALL.equals(agent)) return true; boolean foundUserAgent = false; for (Group group : groups) { Collections.sort(group.getRules()); for (String ua : group.getUserAgents()) { if (agent.contains(ua.toLowerCase())) { foundUserAgent = true; if (checkGroupRules(group, path)) return true; } } if (!foundUserAgent) { for (String ua : group.getUserAgents()) { if (!ALL.equals(ua)) continue; if (checkGroupRules(group, path)) return true; } } } return false; }  boolean canCrawl(String agent, String path, String rules);  }
@Test public void testGroupWithOneAllowRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-one-allow-rule.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/test", content)); assertTrue(ruleMatcher.canCrawl("abc", "/*", content)); }
@Test public void testGroupWithOneDisallowRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-one-disallow-rule.json"); String content = TestUtils.readFile(stream); assertFalse(ruleMatcher.canCrawl("*", "/", content)); assertFalse(ruleMatcher.canCrawl("ab", "/", content)); assertFalse(ruleMatcher.canCrawl("ab", "/test", content)); assertFalse(ruleMatcher.canCrawl("GoogleBot", "", content)); assertFalse(ruleMatcher.canCrawl("GoogleBot", null, content)); }
@Test public void testGroupWithOneEmptyRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-one-empty-rule.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/", content)); assertTrue(ruleMatcher.canCrawl("ab", "/test", content)); assertTrue(ruleMatcher.canCrawl("ab", "/", content)); }
@Test public void testGroupMultiRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-multi-rule.json"); String content = TestUtils.readFile(stream); assertFalse(ruleMatcher.canCrawl("*", "/users/hello", content)); assertFalse(ruleMatcher.canCrawl("*", "/users/", content)); assertFalse(ruleMatcher.canCrawl("*", "/links/source/test", content)); assertFalse(ruleMatcher.canCrawl("*", "/links/source/", content)); assertTrue(ruleMatcher.canCrawl("*", "/hello", content)); assertTrue(ruleMatcher.canCrawl("*", "/links/hello", content)); assertTrue(ruleMatcher.canCrawl("*", "/users/pazkidd/test", content)); assertTrue(ruleMatcher.canCrawl("*", "/users/russelltodd/", content)); }
@Test public void testGroupMultiRuleWithUnknownUserAgent() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-multi-rule.json"); String content = TestUtils.readFile(stream); assertFalse(ruleMatcher.canCrawl("abc-bot", "/users/hello", content)); assertFalse(ruleMatcher.canCrawl("abc-bot", "/users/", content)); assertFalse(ruleMatcher.canCrawl("abc-bot", "/links/source/test", content)); assertFalse(ruleMatcher.canCrawl("abc-bot", "/links/source/", content)); assertTrue(ruleMatcher.canCrawl("abc-bot", "/hello", content)); assertTrue(ruleMatcher.canCrawl("abc-bot", "/links/hello", content)); assertTrue(ruleMatcher.canCrawl("abc-bot", "/users/pazkidd/test", content)); assertTrue(ruleMatcher.canCrawl("abc-bot", "/users/russelltodd/", content)); }
@Test public void testGroupMultiAgentAndRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-multi-agent-and-rule.json"); String content = TestUtils.readFile(stream); assertFalse(ruleMatcher.canCrawl("googlebot", "/users/hello", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/users/", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/links/source/test", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/links/source/", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/test", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/users", content)); assertTrue(ruleMatcher.canCrawl("*", "/links", content)); assertTrue(ruleMatcher.canCrawl("*", "/links/sourc", content)); assertTrue(ruleMatcher.canCrawl("*", "/hello/links/source", content)); assertTrue(ruleMatcher.canCrawl("hello007", "/hello/links/source", content)); assertTrue(ruleMatcher.canCrawl("hello007", "/users/javi", content)); }
@Test public void testCheckUserAgentPriority() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("similar-agents-robots.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("GoogleBot-News", "/users", content)); assertTrue(ruleMatcher.canCrawl("GoogleBot-News", "/hello", content)); assertTrue(ruleMatcher.canCrawl("yanDex", "/test/hello", content)); assertTrue(ruleMatcher.canCrawl("yanDex", "/testing", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/users", content)); assertFalse(ruleMatcher.canCrawl("007", "/Test/hello", content)); assertFalse(ruleMatcher.canCrawl("007", "/hello", content)); }
@Test public void testEmptyUserAgentToBeMatched() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("empty-default-agent.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/users", content)); assertTrue(ruleMatcher.canCrawl("abc", "/users", content)); assertTrue(ruleMatcher.canCrawl("abc", "/", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/hello", content)); assertFalse(ruleMatcher.canCrawl("googlebot", "/", content)); }
@Test public void testOnlyDefaultAgent() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("only-default-agent.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/test", content)); assertTrue(ruleMatcher.canCrawl("*", "/test/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/test/1", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/hello", content)); assertTrue(ruleMatcher.canCrawl("*", "/hello", content)); }
@Test public void testSameAllowAndDisallowDefaultRule() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("same-default-rules.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("googlebot", "/test", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/", content)); }
@Test public void testSameAllowAndDisallowRulePattern() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("same-pattern-rules.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("googlebot", "/test/file.html", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/test", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/not-exists", content)); }
@Test public void testDollarPattern() throws Exception { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("dollar-pattern.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/hello/test", content)); assertTrue(ruleMatcher.canCrawl("*", "/hello/allow/this/rule/test", content)); assertTrue(ruleMatcher.canCrawl("*", "/hello/allow/this/rule/test.php", content)); assertTrue(ruleMatcher.canCrawl("*", "/name/test.php", content)); assertFalse(ruleMatcher.canCrawl("*", "/hello/test%24", content)); assertFalse(ruleMatcher.canCrawl("*", "/hello/test/", content)); assertFalse(ruleMatcher.canCrawl("googleBot", "/hello/Test", content)); assertFalse(ruleMatcher.canCrawl("007", "/test/", content)); assertFalse(ruleMatcher.canCrawl("*", "/hello/allow/this/rule/test.PHP", content)); assertFalse(ruleMatcher.canCrawl("*", "/hello/allow/this/rule/test.php5", content)); assertFalse(ruleMatcher.canCrawl("*", "/hello/allow/this/rule/test.php/", content)); }
@Test public void testAllWildcardPattern() throws Exception { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("wildcard-disallow-all.json"); String content = TestUtils.readFile(stream); assertFalse(ruleMatcher.canCrawl("*", "/hello?user=javi", content)); assertFalse(ruleMatcher.canCrawl("*", "/disallow/this/rule?user=javi", content)); assertTrue(ruleMatcher.canCrawl("007", "/test?user=javi", content)); assertTrue(ruleMatcher.canCrawl("007", "/test/allow/this/hello", content)); }
@Test public void testCustom1() throws Exception { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("custom-1.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("googlebot", "/this-is-a-test.html", content)); assertTrue(ruleMatcher.canCrawl("Googlebot/2.1", "/this-is-a-test.html", content)); assertTrue(ruleMatcher.canCrawl("googlebot", "/this-is-a-test.html", content)); assertTrue(ruleMatcher.canCrawl("AdsBot-Google", "/this-is-a-test.html", content)); assertTrue(ruleMatcher.canCrawl("AdsBot-Google-Mobile", "/this-is-a-test.html", content)); assertFalse(ruleMatcher.canCrawl("voltron", "/this-isnt-allowed?directory=empty", content)); assertFalse(ruleMatcher.canCrawl("voltron", "/customer/javi", content)); }
@Test public void testFixIssue33() throws Exception { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("comparison-violates-contract.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("googlebot", "/", content)); assertTrue(ruleMatcher.canCrawl("curl/7.2.1", "/modules/hello-world.js", content)); assertFalse(ruleMatcher.canCrawl("httpie/1.2.3", "/modules/not-allowed", content)); }
@Test public void testNoGroupsFound() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("empty-group.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/test", content)); }
@Test public void testPathIsRobotsFile() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-without-rules.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/robots.txt", content)); assertTrue(ruleMatcher.canCrawl("abc", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/test", content)); assertTrue(ruleMatcher.canCrawl("*", "/", content)); }
@Test public void testGroupWithEmptyRules() throws IOException { InputStream stream = this.getClass().getClassLoader().getResourceAsStream("group-without-rules.json"); String content = TestUtils.readFile(stream); assertTrue(ruleMatcher.canCrawl("*", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/", content)); assertTrue(ruleMatcher.canCrawl("abc", "/test", content)); }