Added the ability to parse additional bots.
e.g.,
BacklinkCrawler (http://www.backlinktest.com/crawler.html)
Scrapy/1.0.1 ( http://scrapy.org)
@@ -439,18 +439,18 @@ parse_crawler (char *str, char *match, char *type) {
while (match != str) {
match--;
- if (*match == ' ' || *match == '+') {
+ if (*match == ' ' || *match == '+' || match == str) {
found = 1;
break;
}
}
/* same addr */
- if (match == str)
+ if (!found && match == str)
return NULL;
/* account for the previous +|space */
- if (found)
+ if (found && match != str)
match++;
if ((ptr = strpbrk (match, "; ")))
