Scrapy blank AssertionError?
Question: The below code is throwing the following error for each request sent to the
parse method (Scrapy v0.24.4):

    
    
    2014-12-30 01:20:06+0000 [yelp_spider] DEBUG: Crawled (200) <GET http://www.yelp.com/biz/lookout-tavern-oak-bluffs> (referer: http://www.yelp.com/search?find_desc=Restaurants&find_loc=02557&ns=1) ['partial']
    2014-12-30 01:20:06+0000 [yelp_spider] ERROR: Spider error processing <GET http://www.yelp.com/biz/lookout-tavern-oak-bluffs>
            Traceback (most recent call last):
              File "/usr/lib/python2.7/site-packages/scrapy/core/scraper.py", line 111, in _scrape_next
                self._scrape(response, request, spider).chainDeferred(deferred)
              File "/usr/lib/python2.7/site-packages/scrapy/core/scraper.py", line 118, in _scrape
                dfd = self._scrape2(response, request, spider) # returns spiders processed output
              File "/usr/lib/python2.7/site-packages/scrapy/core/scraper.py", line 128, in _scrape2
                request_result, request, spider)
              File "/usr/lib/python2.7/site-packages/scrapy/core/spidermw.py", line 69, in scrape_response
                dfd = mustbe_deferred(process_spider_input, response)
            --- <exception caught here> ---
              File "/usr/lib/python2.7/site-packages/scrapy/utils/defer.py", line 39, in mustbe_deferred
                result = f(*args, **kw)
              File "/usr/lib/python2.7/site-packages/scrapy/core/spidermw.py", line 48, in process_spider_input
                return scrape_func(response, request, spider)
              File "/usr/lib/python2.7/site-packages/scrapy/core/scraper.py", line 138, in call_spider
                dfd.addCallbacks(request.callback or spider.parse, request.errback)
              File "/usr/lib64/python2.7/site-packages/twisted/internet/defer.py", line 288, in addCallbacks
                assert callable(callback)
            exceptions.AssertionError:
    

Code:

    
    
    import scrapy
    from scrapy import Request
    import re
    
    ROOT_URL = "http://www.yelp.com"
    
    class YelpReview(scrapy.Item):
        zip_code = scrapy.Field()
        review_date = scrapy.Field()
    
    class yelp_spider(scrapy.Spider):
        name = 'yelp_spider'
        allowed_domains = ['yelp.com']
        start_urls = ["http://www.yelp.com/search?find_desc=Restaurants&find_loc=02557&ns=1"]
    
        def parse(self, response):
            business_urls = [business_url.extract() for
                    business_url in response.xpath('//a[@class="biz-name"]/@href')[1:]
                    ]
            for business_url in business_urls:
                yield Request(url=ROOT_URL + business_url, callback="scrape_reviews")
    
            if response.url.find('?start=') == -1:
                self.createRestaurantPageLinks(response)
    
        def scrape_reviews(self, response):
            reviews = response.xpath('//meta[@itemprop="datePublished"]/@content')
            item = YelpReview()
    
            for review in reviews:
                item['zip_code'] = "02557"
                item['review_date'] = review.extract()
                yield item
    
            if response.url.find('?start=') == -1:
                self.createReviewPageLinks(response)
    
        def createRestaurantPageLinks(self, response):
            raw_num_results = response.xpath('//span[@class="pagination-results-window"]/text()').extract()[0]
            num_business_results = int(re.findall(" of (\d+)", raw_num_results)[0])
            BUSINESSES_PER_PAGE = 10
            restaurant_page_links = [Request(url=response.url + '?start=' + str(BUSINESSES_PER_PAGE*(n+1)),
                callback="parse") for n in range(num_business_results/BUSINESSES_PER_PAGE)]
    
            return restaurant_page_links
    
        def createReviewsPageLinks(self, response):
            REVIEWS_PER_PAGE = 40
            num_review_results = int(response.xpath('//span[@itemprop="reviewCount"]/text()').extract()[0])
            review_page_links = [Request(url=response.url + '?start=' + str(REVIEWS_PER_PAGE*(n+1)),
                callback="scrape_reviews") for n in range(num_review_results/REVIEWS_PER_PAGE)]
    
            return review_page_links
    

I've tried making a bunch of changes but still can't figure out what's
triggering this error.


Answer: You need to return from the `parse()` method:

    
    
    if response.url.find('?start=') == -1:
        return self.createRestaurantPageLinks(response)
    

