title: Clicking on read more by using scrapy
body: <p>I have been working on a scrapy spider that is now capable of scraping tripadvisor.com reviews. However, to extract the trip type I need to get my spider to press on 'read more...'. Otherwise the coding does not show up. For the first page (the first 5 reviews), the trip types are still hidden in the code, but by going to the next review page the code completely disappears and it only returns N/A. I have been seeing some previous questions being answered by using Selenium, but after trying I haven't been able to get it working. My code is as follows (without any coding from selenium). Thanks in advance!</p>
  
  <pre><code>    # -*- coding: utf-8 -*-
  import scrapy
  import lxml.html
  
  class TripreviewsSpider(scrapy.Spider):
      name = 'tripreviews'
      allowed_domains = ['tripadvisor.com']
      start_urls = ['https://www.tripadvisor.com/Hotels-g60763-New_York_City_New_York-Hotels.html']
  
  
          ### Listing page crawling
      def parse_listing(self, response):
  
           urls = response.css('a.property_title.prominent::attr(href)').extract()
           for urlHotel in urls:
               print(urlHotel)
               urlHotel = response.urljoin(urlHotel)
               yield scrapy.Request(url = urlHotel ,callback=self.parse)  ###If scrapy recognises a detail page it opens it and will do a callback to line 23 (parse_review).
  
           next_listing_url = response.css('a.nav.next.ui_button.primary.cx_brand_refresh_phase2::attr(href)').extract()  ### If scrapy does not recognise a detail page, it will go to the next listing page and starts at line 11 again.
           next_listing_url = response.urljoin(next_listing_url)
           yield scrapy.Request(url=next_listing_url, callback=self.parse_listing)
  
      def parse(self, response):
          self.log('I just visited:' + response.url)
  
          #Set up default value
          defaultValue = "N/A"
  
          #Narrow review dataset
          foundReviews = response.css('div[data-test-target="reviews-tab"] &gt; div[data-test-target="HR_CC_CARD"]')
  
          #Narrow hoteldetails dataset
          foundHotelDetails = response.css("div[class*='ui_column is-12-tablet is-9-mobile hotels-hotel-review-atf-info-parts-ATFInfo__description--']")
  
          #listingName
          listingName = []
  
          for hotel in foundHotelDetails:
              containslistingName = hotel.css("div[class*='ui_column is-12-tablet is-9-mobile hotels-hotel-review-atf-info-parts-ATFInfo__description--'] &gt; div &gt; h1#HEADING::text").extract() 
  
              listingName.append(containslistingName or defaultValue)
  
          listingRating = []
  
          for ratingAverage in foundHotelDetails.css("a[class*='hotels-hotel-review-atf-info-parts-Rating__ratingsAnchor--'] &gt; span[class*='bubble']").xpath("@class").extract():
              #For each found rating: strip everything off (extra classes and the prefix 'bubble_') to get the rating value.
              #Divide it by 10 to get the proper rating.
              listingRating.append( int(ratingAverage.split(" ")[1].split("bubble_")[1]) / 10 )
  
          listingCategory = []
  
          for category in foundHotelDetails:
              containsCategory = category.css("div[class*='hotels-hotel-review-atf-info-parts-PopIndex__popIndex--'] &gt; span &gt; a::text").get()
              listingCategory.append(containsCategory.split(" ")[0] or defaultValue)
  
          #Build list of review attributes using narrowed dataset
          reviewTitle = foundReviews.css('div[data-test-target="review-title"] &gt; a &gt; span &gt; span::text').extract()
  
          #Clean up html tags in review text
          formattedReviewText = []
  
          for text in foundReviews:
              #For each entry: check if it has an element containing the trip type
              foundText = str(text.css("q[class*='location-review-review-list-parts-ExpandableReview__reviewText']").get())
              formattedText = lxml.html.fromstring(foundText).text_content()
  
              #Add the type if it was found in text, or add the default value
              formattedReviewText.append(formattedText or defaultValue)
  
          reviewDateofStay = foundReviews.css("span[class*='location-review-review-list-parts-EventDate__event_date']::text").extract()
  
          #Because the origin country of the reviewer isn't always present, create a specific list.
          #This list makes sure that if trip type is present, it will be added to the corresponding review.
          #Otherwise it's defaultValue
          #That way the list lengths will always match.
          reviewLocation = []
          for location in foundReviews:
              #For each entry: check if it has an element containing the trip type
              containsReviewerOrigin = location.css("span[class*='default social-member-common-MemberHometown__hometown']::text").get()
  
              #Add the type if it was found in text, or add the default value
              reviewLocation.append(containsReviewerOrigin or defaultValue)
  
          #Because ratings are immediately turned into a css class, you cannot extract this immediately.
          #You need to strip the exact class out and then divide by 10 so you can get the rating in a scale from 1.0 to 5.0.
          #Ratings are also optional....
          reviewRating = []
          for rating in foundReviews.css("div[data-test-target='review-rating'] &gt; span[class*='bubble']").xpath("@class").extract():
              #For each found rating: strip everything off (extra classes and the prefix 'bubble_') to get the rating value.
              #Divide it by 10 to get the proper rating.
              reviewRating.append( int(rating.split(" ")[1].split("bubble_")[1]) / 10 )
  
          #Because trip type isn't always present, create a specific list.
          #This list makes sure that if trip type is present, it will be added to the corresponding review.
          #Otherwise it's defaultValue
          #That way the list lengths will always match.
  
          reviewTripType = []
  
          for review in foundReviews:
              #For each entry: check if it has an element containing the trip type
              containsTripType = review.css("span[class*='location-review-review-list-parts-TripType__trip_type--']::text").get()
              #Add the type if it was found in text, or add the default value
              reviewTripType.append(containsTripType or defaultValue)
  
          reviewHelpfulVotes = []
  
          for helpfulVotes in foundReviews:
              #For each entry: check if it has an element containing the trip type
              hasHelpfulVotes = helpfulVotes.css("div[class*='location-review-social-bar-SocialBar__bar--'] &gt; span[class*='social-statistics-bar-SocialStatisticsBar__bar--'] &gt; span[class*='social-statistics-bar-SocialStatisticsBar__counts']::text").get()
              #Add the type if it was found in text, or add the default value. Check if hasHelpfulVotes isn't Nonetype.
              #And split value on first space and get first element because default value does not contain spaces it will never look weird and will always make sure that validHelpfulVotes values are always split correctly.
              validHelpfulVote = hasHelpfulVotes or defaultValue
              reviewHelpfulVotes.append(validHelpfulVote.split(" ")[0])
  
          reviewPrefix = {
              'name': listingName,
              'listing Rating': listingRating,
              'category': listingCategory
          }
  
  
          #Create the review data
          for review in zip(reviewLocation, reviewTitle, formattedReviewText, reviewRating, reviewDateofStay, reviewTripType, reviewHelpfulVotes):
  
              reviewData = {
  
                  'Origin of reviewer': review[0],
                  'Title': review[1],
                  'ReviewText': review[2],
                  'Rating': review[3] or defaultValue,
                  'Date of stay': review[4] or defaultValue,
                  'Trip type': review[5],
                  'Helpful votes': review[6]
              }
              reviewTotal = {**reviewPrefix,**reviewData}
              yield reviewTotal
          ### Next review page
          nextReviewPage = response.css('a.ui_button.nav.next.primary::attr(href)').extract_first()
          if nextReviewPage:
              nextReviewPage = response.urljoin(nextReviewPage)
              print(nextReviewPage)  #For checking purposes
              yield scrapy.Request(url = nextReviewPage, callback = self.parse)
          yield scrapy.Request(url = 'https://www.tripadvisor.com/Hotels-g60763-New_York_City_New_York-Hotels.html', callback = self.parse_listing)
  </code></pre>