title: how to use scrapy with the crawler template and scrapy-splash to parse javascript
body: <p>i am trying to scrape amazon's products using scrapy with the crawl template, but i found the amazon use some javascript to get some block of the product details, so i decide to use splash to render javascript, it works fine in the shell command, but i can't figure out how to implement it in my code.</p>
  <pre><code>import scrapy
  from scrapy.linkextractors import LinkExtractor
  from scrapy.spiders import CrawlSpider, Rule
  
  
  class AmazonCrawlerSpider(CrawlSpider):
      name = 'amazon_Crawler'
      allowed_domains = ['amazon.com']
      start_urls = ['https://www.amazon.com/s?i=specialty-aps&amp;bbn=16225009011&amp;rh=n%3A%2116225009011%2Cn%3A502394&amp;ref=nav_em__nav_desktop_sa_intl_camera_and_photo_0_2_5_3']
  
      len_product_details = LinkExtractor(restrict_css='h2 &gt; a')
      product_details = Rule(len_product_details,
                             callback='parse_item', follow=False)
  
      len_products_pagination = LinkExtractor(
          restrict_xpaths='//*[@id=&quot;search&quot;]/div[1]/div[1]/div/span[3]/div[2]/div[37]/div/div/span/a[3]')
      products_pagination = Rule(len_products_pagination, follow=True)
      rules = (
          product_details, products_pagination
      )
  
      def parse_item(self, response):
  
          data = {
  
              &quot;categorie_0&quot;: response.xpath('//*[@id=&quot;wayfinding-breadcrumbs_feature_div&quot;]/ul/li[1]/span/a/text()').get(),
              &quot;categorie_1&quot;: response.xpath('//*[@id=&quot;wayfinding-breadcrumbs_feature_div&quot;]/ul/li[3]/span/a/text()').get(),
              &quot;title&quot;: response.css('h1 &gt; span ::text').get(),
              &quot;price&quot;: response.xpath('//div[@id=&quot;corePrice_feature_div&quot;]/div/span/span[1]//text()').get(),
              &quot;amazon_globale_shiping&quot;: response.xpath('//*[@id=&quot;a-popover-content-2&quot;]/table/tbody/tr[2]/td[3]/span/text()').get(),
              &quot;estimated_import_fees_deposit&quot;: response.xpath('//*[@id=&quot;a-popover-content-2&quot;]/table/tbody/tr[3]/td[3]/span/text()').get(),
              &quot;total&quot;: response.xpath('//*[@id=&quot;a-popover-content-2&quot;]/table/tbody/tr[5]/td[3]/span/text()').get(),
              &quot;delevery_period&quot;: response.xpath('//*[@id=&quot;mir-layout-DELIVERY_BLOCK-slot-PRIMARY_DELIVERY_MESSAGE_LARGE&quot;]/span/span/text()').get(),
              &quot;delevery_destination&quot;: response.xpath('//*[@id=&quot;contextualIngressPtLabel_deliveryShortLine&quot;]/span[2]/text()').get(),
              &quot;in_stock&quot;: response.xpath('//*[@id=&quot;availability&quot;]/span/text()').get(),
              &quot;quantity&quot;: &quot;not_exist&quot;,
              &quot;ship_from&quot;: response.xpath('//*[@id=&quot;tabular-buybox&quot;]/div[1]/div[2]/div/span/text()').get(),
              &quot;sold_by&quot;: {
                  &quot;name&quot;: response.xpath('//*[@id=&quot;sellerProfileTriggerId&quot;]/text()').get(),
                  'store_url': response.xpath('//*[@id=&quot;sellerProfileTriggerId&quot;]/@href').get(),
                  'packaging': response.xpath('//*[@id=&quot;tabular-buybox&quot;]/div[1]/div[6]/div/span/text()').get()
              },
              &quot;description&quot;: response.xpath('//*[@id=&quot;productDescription&quot;]/p/text()').get(),
              # &quot;brand&quot;: response.xpath('//*[@id=&quot;productOverview_feature_div&quot;]/div/table/tbody/tr[1]/td[2]/span/text()').get(),
              &quot;is_returned&quot;: response.xpath('//*[@id=&quot;productSupportAndReturnPolicy-return-policy-popover-celWidget&quot;]/div/div[1]/text()').get(),
              &quot;extra_info&quot;: [],
              &quot;details&quot;: [],
              &quot;about_this_item&quot;: [],
              &quot;note&quot;: response.xpath('//*[@id=&quot;universal-product-alert&quot;]/div/span[2]/text()').get(),
              &quot;Q_AW&quot;: [],
              &quot;Customer_reviews&quot;: {
                  &quot;customer_rate&quot;: response.xpath('//*[@id=&quot;reviewsMedley&quot;]/div/div[1]/div[2]/div[1]/div/div[2]/div/span/span/text()').get(),
                  &quot;total_rate&quot;: response.xpath('//*[@id=&quot;reviewsMedley&quot;]/div/div[1]/div[2]/div[2]/span/text()').get(),
                  &quot;global_rate&quot;: {
                      &quot;1_star&quot;: response.xpath('//*[@id=&quot;histogramTable&quot;]/tbody/tr[5]/td[3]/span[2]/a/text()').get(),
                      &quot;2_star&quot;: response.xpath('//*[@id=&quot;histogramTable&quot;]/tbody/tr[4]/td[3]/span[2]/a/text()').get(),
                      &quot;3_star&quot;: response.xpath('//*[@id=&quot;histogramTable&quot;]/tbody/tr[3]/td[3]/span[2]/a/text()').get(),
                      &quot;4_star&quot;: response.xpath('//*[@id=&quot;histogramTable&quot;]/tbody/tr[2]/td[3]/span[2]/a/text()').get(),
                      &quot;5_star&quot;: response.xpath('//*[@id=&quot;histogramTable&quot;]/tbody/tr[1]/td[3]/span[2]/a/text()').get(),
                  },
                  &quot;rate_by_feature&quot;: [],
                  &quot;product_reviews&quot;: []
  
              },
              &quot;url&quot;: response.url
  
          }
          for reveiw in response.xpath('//*[@id=&quot;cm-cr-dp-review-list&quot;]/div'):
              data[&quot;Customer_reviews&quot;][&quot;product_reviews&quot;].append(
                  {
                      &quot;rate&quot;: reveiw.xpath('/div/div/div[2]/a/i/span/text()').get(),
                      &quot;feature&quot;: reveiw.xpath('div/div/div[2]/a[2]/span/text()').get(),
                      &quot;date_from&quot;: reveiw.xpath('div/div/span/text()').get(),
                      &quot;verified&quot;: reveiw.xpath('div/div/div[3]/span[2]/text()').get(),
                      &quot;review&quot;: reveiw.xpath('div/div/div[4]/span/div/div[1]/span/text()').get(),
                      'view_reaction': reveiw.xpath('div/div/div[5]/span[1]/div[1]/span/text()').get()
                  }
              )
  
          for cr_rf in response.xpath('//*[@id=&quot;cr-summarization-attributes-list&quot;]/div'):
              data[&quot;Customer_reviews&quot;][&quot;rate_by_feature&quot;].append(
                  {
                      &quot;key&quot;: cr_rf.xpath('div/div/div/div/span/text()').get(),
                      &quot;value&quot;: response.xpath('div/div/div[2]/span[2]/text()').get()
                  }
              )
  
          for Q_AW in response.xpath('//*[@id=&quot;ask-btf-container&quot;]/div/div/div[2]/span/div/div'):
              data[&quot;Q_AW&quot;].append(
                  {
                      &quot;Question&quot;: Q_AW.xpath('div/div[2]/div/div/div[2]/a/span/text()').get(),
                      &quot;Answer&quot;:  Q_AW.xpath('div/div[2]/div[2]/div/div[2]/span/span[2]/text()').get(),
                      &quot;vote&quot;: Q_AW.xpath('div/div/ul/li[2]/span[1]/text()').get(),
                      &quot;date_answer&quot;: Q_AW.xpath('div/div[2]/div[2]/div/div[2]/span[3]/text()').get()
                  }
              )
  
          for extra_info in response.xpath('//*[@id=&quot;productDetails_detailBullets_sections1&quot;]/tbody/tr'):
              data[&quot;extra_info&quot;].append(
                  {
                      &quot;1&quot;: extra_info.css('th::text').get(),
                      &quot;2&quot;: extra_info.css('td::text').get()
                  }
              )
          for index, about_this_item in enumerate(response.xpath('//*[@id=&quot;feature-bullets&quot;]/ul/li')):
              data[&quot;about_this_item&quot;].append(
                  {
                      index+1: about_this_item.xpath('span/text()').get(),
  
                  }
              )
          for extra in response.xpath('//*[@id=&quot;productOverview_feature_div&quot;]/div/table/tbody/tr'):
              data['details'].append(
                  {
                      extra.xpath('td[1]/span/text()').get(): extra.css('td[2]/span/text()').get()
                  }
              )
  
          yield data
  
  </code></pre>