'join()
Returns a deferred that is fired when all managed :attr:`crawlers` have
completed their executions.'

@defer.inlineCallbacks
def join(self):

    while self._active:
        (yield defer.DeferredList(self._active))
