'Called when a spider gets idle. This function is called when there
are no remaining pages to download or schedule. It can be called
multiple times. If some extension raises a DontCloseSpider exception
(in the spider_idle signal handler) the spider is not closed until the
next loop and this function is guaranteed to be called (at least) once
again for this spider.'

def _spider_idle(self, spider):

    res = self.signals.send_catch_log(signal=signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
    if any(((isinstance(x, Failure) and isinstance(x.value, DontCloseSpider)) for (_, x) in res)):
        return
    if self.spider_is_idle(spider):
        self.close_spider(spider, reason='finished')
