'Override this method to implement a different offsite policy'

def get_host_regex(self, spider):

    allowed_domains = getattr(spider, 'allowed_domains', None)
    if (not allowed_domains):
        return re.compile('')
    regex = ('^(.*\\.)?(%s)$' % '|'.join((re.escape(d) for d in allowed_domains if (d is not None))))
    return re.compile(regex)
