# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re

header={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'}


# ## Function to scrape product name and asin no (as the file path is similar)

# +
def Search(search_query):
    url="https://www.amazon.in/s?k="+search_query #search link
    page=requests.get(url,headers=header)  
    if page.status_code==200:
        return page                                #returns the page if there is no error
    else:
        return "Error"


# -

# ## Function to scrape link of the All customer reviews to acess all the reviews

def url_link(query):
    url="https://www.amazon.in/dp/"+query     #search link
    page=requests.get(url,headers=header)
    if page.status_code==200:
        return page                            #return the page if there is no error
    else:
        return "Error"    


# ## Function to scrape product reviews 

def content(query):
    url="https://www.amazon.in/"+query
    page=requests.get(url,headers=header)
    if page.status_code==200:
        return page
    else:
        return "Error" 


# ## This saves all the product name and the asin no in a list

product_Name=[]
Asin_no=[]
for i in range(1,16):
    response=Search("smartphones&page="+str(i))
    soup=BeautifulSoup(response.content)
    for i in soup.findAll('span',attrs={'class':'a-size-medium a-color-base a-text-normal'}): 
        product_Name.append(i.text)
    for i in soup.find_all('div',attrs={'data-asin':True}):
        Asin_no.append(i['data-asin'])

product_Name

Asin_no

# ## This takes all the links and ignores if the asin no is blank

Link=[]
for i in range(len(Asin_no)):
    if Asin_no[i]!='':
        link_response=url_link(Asin_no[i])       #iterates through Asin no to access the product
        soup=BeautifulSoup(link_response.content)
        for i in soup.findAll('a',attrs={'data-hook':'see-all-reviews-link-foot'}):
            Link.append(i['href'])               #saves the footer link of all reviews to a list

len(Link)

# ## For each product we are scraping 15 pages of reviews and ratings

reviews=[]
ratings=[]
for k in range(len(Link)):                                      #helps to iterate for different products
    for i in range(1,15):                               #each product , it scrapes 15 pages of reviews
        cont_response=content(Link[k]+'&pageNumber='+str(i))    #iterates through multiple pages of the reviews
        soup=BeautifulSoup(cont_response.content)
        for j in soup.findAll("span",attrs={'data-hook':'review-body'}):
            reviews.append(j.text)                        #saves review content
        for star in soup.findAll("i",attrs={'data-hook':"review-star-rating"}):
            ratings.append(star.text)                 #saves ratings of the respective reviews
            

# ## saving the file in dataframe

new_reviews = []
for review in reviews:
    new_reviews.append(review.strip())

df=pd.DataFrame(columns=['Smartphone_Review','Rating']) 


df.to_csv('Smartphone_Review.csv',index=False)   #saving it into a dataframe

df=df.drop_duplicates()                   #removing duplicates

df['Smartphone_Review']=new_reviews
df['Rating']=ratings

df



StarcoderdataJupyterScriptsDedupFiltered