Scrapy SgmlLinkExtractor - Having trouble with recursively scraping
Question: **Update: Apparently I can't answer my own question within 8 hours, but I got
it to work. Thanks guys!**

I am having trouble getting scrapy to crawl the links on the start_url.

The following is my code below:

    
    
    from scrapy.selector import HtmlXPathSelector
    from scrapy.contrib.spiders import CrawlSpider, Rule
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    
    from dirbot.items import Website
    
    
    class mydomainSpider(CrawlSpider):
        name = "mydomain"
        allowed_domains = ["mydomain.com"]
        start_urls = ["http://www.mydomain.com/cp/133162",]
    
        """133162 category to crawl"""
    
        rules = (
            Rule(SgmlLinkExtractor(allow=('133162', ), deny=('/ip/', ))),
        )
    
        def parse(self, response):
            hxs = HtmlXPathSelector(response)
            sites = hxs.select('//html')
            items = []
    
            for site in sites:
                item = Website()
                item['description'] = site.select('//meta[@name="Description"]/@content').extract()
                item['url'] = response.url
                item['title'] = site.xpath('/html/head/title/text()').extract()
                items.append(item)
    
            return items
    

I am new to python and am open to any advice. Thank you for your time!


Answer: I got it to work, thanks guys!

    
    
    from scrapy.selector import HtmlXPathSelector
    from scrapy.contrib.spiders import CrawlSpider, Rule
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    
    from wallspider.items import Website
    
    
    class mydomainSpider(CrawlSpider):
        name = "mydomain"
        allowed_domains = ["www.mydomain"]
        start_urls = ["http://www.mydomain/cp/133162",]
    
        rules = (Rule (SgmlLinkExtractor(allow=('133162', ),deny=('/ip/', 'search_sort=', 'ic=60_0', 'customer_rating', 'special_offers', ),)
        , callback="parse_items", follow= True),
        )
    
        def parse_items(self, response):
            hxs = HtmlXPathSelector(response)
            sites = hxs.select('//*')
            items = []
    
            for site in sites:
                item = Website()
                item['referer'] = response.request.headers.get('Referer')
                item['url'] = response.url
                item['title'] = site.xpath('/html/head/title/text()').extract()
                item['description'] = site.select('//meta[@name="Description"]/@content').extract()
                items.append(item)
    
            return items
    

