title: Scraping in Python - Trying to get URL for next page
body: <p>I have been tried to scrape this <a href="https://ba.olx.com.br/grande-salvador/salvador/imbui/imoveis/venda/apartamentos?pe=280000&amp;ps=200000&amp;ros=2" rel="nofollow noreferrer">Olx ad webpage</a> with Python, but I cant make it to find the link for next page, because <code>h.xpath</code> returns empty array. I think the XPath expressions is probably wrong. Could someone help?</p>
  <p>XPath expression:</p>
  <pre><code>    next_page2 = h.xpath('//Div[@class=&quot;sc-hmzhuo inUUEM sc-jTzLTM iwtnNi&quot;]/a/@href')
      print(next_page2)
  </code></pre>
  <p>Full code (.py):</p>
  <pre><code>#https://medium.com/@henriquecoura_87435/webscraping-com-python-extraindo-dados-de-um-ecommerce-89c16b622f69
  
  import lxml.html as parser
  import csv
  import requests
  import time
  import os.path
  
  MAX_PAGES = 3 #float('inf')
  MAX_ADS =  100 #float('inf')
  DELAY_INTERVAL = 5 #seconds
  
  headers = {'User-Agent': '&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36&quot;'}
  start_urls = [
  #    &quot;https://pe.olx.com.br/grande-recife/recife/pina/imoveis/aluguel/apartamentos?bas=2&amp;pe=2500&amp;ps=1000&amp;q=suite&amp;ros=2&quot;, #2500-1000, 2/4, suite, Pina
  #    &quot;https://pe.olx.com.br/grande-recife/recife/espinheiro/imoveis/aluguel/apartamentos?bas=2&amp;pe=2500&amp;ps=1000&amp;q=suite&amp;ros=2&quot;, #2500-1000, 2/4, suite, Espinheiro
  #    &quot;https://pe.olx.com.br/grande-recife/recife/imoveis/aluguel/apartamentos?bas=2&amp;pe=2500&amp;ps=1000&amp;q=suite&amp;ros=2&amp;sd=3743&amp;sd=3771&amp;sd=3770&amp;sd=3751&quot;, #2500-1000, 2/4, suite, Boa Vista, Torreão, Derby, Santo Amaro
  #    &quot;https://pe.olx.com.br/grande-recife/recife/gracas/imoveis/aluguel/apartamentos?bas=2&amp;pe=2500&amp;ps=1000&amp;q=suite&amp;ros=2&quot;, #2500-1000, 2/4, suite, Graças
      &quot;https://ba.olx.com.br/grande-salvador/salvador/imbui/imoveis/venda/apartamentos?pe=280000&amp;ps=200000&amp;ros=2&quot; #2500-1000, 2/4, suite, Boa Viagem
  ]
  
  csv_dic = {'index': '', 'title': '', 'price': '', 'description': '',  'link': '', 'CEP': '', 'Município': '', 'Bairro': '', 'Logradouro': ''}
  
  '''
  # Crawlear a primeira página
  r = requests.get(start_url, headers=headers)
  h = parser.fromstring(r.text)
  links += h.xpath('//li[@class=&quot;sc-1fcmfeb-2 ggOGTJ&quot;]/a/@href') # returns the links for all ads
  print(&quot;Page 1: &quot; + start_url)
  print(&quot;Links for all ads on the page 1&quot;)
  print(&quot;Links: &quot; + links)
  print(&quot;How many links: &quot; + len(links))
  print(&quot;&quot;)
  
  # A próxima página na página inicial está no index 0, nao tem página anterior
  next_page = h.xpath('//li[@class=&quot;sc-1m4ygug-0 ilgdSS&quot;]/a/@href')[0] # returns the link for next page
  '''
  ads = 0
  j = 1
  for start_url in start_urls:
      i = 1
      links = []
      print(&quot;Crawling page by page: &quot; + str(j))
  
      next_page = start_url
      while next_page:
  
          r = requests.get(next_page, headers=headers)
          h = parser.fromstring(r.text)
  
          next_page2 = h.xpath('//Div[@class=&quot;sc-hmzhuo inUUEM sc-jTzLTM iwtnNi&quot;]/a/@href')
          print(next_page2)
  
          '''
          catch = h.xpath('//li[@class=&quot;sc-1fcmfeb-2 juiJqh&quot;]/a/@href')
          print(&quot;Page: &quot; + str(i) + &quot;: &quot; + next_page)
          print(&quot;Links for all ads on the page &quot; + str(i))
          print(catch)
          print(&quot;How many ads: &quot; + str(len(catch)))
          links += catch
          print(&quot;How many ads in total: &quot; + str(len(links)))
          print(&quot;&quot;)
          
          try:
  #            next_page = h.xpath('//li[@class=&quot;sc-1m4ygug-0 ilgdSS&quot;]/a/@href')[0]
              next_page = h.xpath('//Div[@class=&quot;sc-hmzhuo kJjuHR sc-jTzLTM iwtnNi&quot;]/a/@href')[0]
              print(&quot;Next page: &quot; + next_page)
              print(&quot;&quot;)
          # Se tivermos um IndexError é porque estamos na última página
          # não existe mais uma next_page
          # Interrompa o loop
          except IndexError as e:
              next_page = None
              print(&quot;except 1&quot;)
          if i &gt;= MAX_PAGES:
              next_page = None
              print(&quot;except 2&quot;)
          i += 1
          time.sleep(DELAY_INTERVAL)  # Delays for 5 seconds
          '''
  '''
      i=1
      print(&quot;Scraping ad/link by ad/link&quot;)
      print(&quot;How many ads to scrape: &quot; + str(len(links)))
      for link in links:
          ads += 1
          towrite = []
          towrite_h = []
          csv_dic.clear()
          print(&quot;Ad/link spraping &quot; + str(i) + &quot;: &quot; + link)
  
          r = requests.get(link, headers=headers)
          h = parser.fromstring(r.text)
  
          csv_dic[&quot;index&quot;] = ads
  
          title = h.xpath('//h1[@class=&quot;sc-45jt43-0 eCghYu sc-ifAKCX cmFKIN&quot;]/text()')[0]
          csv_dic[&quot;title&quot;] = title
          print(&quot;title: &quot; + title)
  
          price = h.xpath('//h2[@class=&quot;sc-1wimjbb-0 JzEH sc-ifAKCX cmFKIN&quot;]/text()')[0]
          csv_dic[&quot;price&quot;] = price
          print(&quot;price: &quot; + price)
  
          description = h.xpath('//span[@class=&quot;sc-1sj3nln-1 eOSweo sc-ifAKCX cmFKIN&quot;]/text()')[0].replace('\n','')
          csv_dic[&quot;description&quot;] = description
          print(&quot;description: &quot; + description)
  
          csv_dic[&quot;link&quot;] = link
          print(&quot;link: &quot; + link)
  
          loc = h.xpath('//div[@class=&quot;sc-bwzfXH h3us20-0 cBfPri&quot;]//dd[@class=&quot;sc-1f2ug0x-1 ljYeKO sc-ifAKCX kaNiaQ&quot;]/text()')
          loc.insert(0, &quot;&quot;)
          loc.insert(0, &quot;&quot;)
          loc.insert(5, &quot;&quot;)
          print(loc)
          loc_keys = h.xpath('//div[@class=&quot;sc-bwzfXH h3us20-0 cBfPri&quot;]//dt[@class=&quot;sc-1f2ug0x-0 cLGFbW sc-ifAKCX cmFKIN&quot;]/text()')
          print(loc_keys)
          v = 0
          for key in loc_keys:
              csv_dic[key] = loc[v]
              v += 1
          print(&quot;loc: &quot;)
          print(loc)
  
          towrite.extend(list(csv_dic.values()))
          towrite_h.extend(list(csv_dic.keys()))
  
          if os.path.exists('Olx.csv'):
              with open('Olx.csv', 'a', newline='') as file:
                  writer = csv.writer(file)
                  writer.writerow(towrite)
          else:
              with open('Olx.csv', 'w', newline='') as file:
                  writer = csv.writer(file)
                  writer.writerow(towrite_h)
                  writer.writerow(towrite)
  
          if i &gt;= MAX_ADS:
              break
          i += 1
          time.sleep(DELAY_INTERVAL)  # Delays for 5 seconds
      j += 1
      print(&quot;&quot;)
  '''
  print(&quot;Total of ads visited: &quot; + str(ads))
  </code></pre>