Spaces:
Sleeping
Sleeping
File size: 3,005 Bytes
7901fc5 01060ba 87b5120 7901fc5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import requests
from bs4 import BeautifulSoup
import pandas as pd
custom_headers = {
"Accept-language": "en-GB,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Cache-Control": "max-age=0",
"Connection": "keep-alive",
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
}
def get_soup(url):
response = requests.get(url, headers=custom_headers)
if response.status_code != 200:
print("Error in getting webpage")
print(f"Error: {response.status_code} - {response.reason}")
exit(-1)
soup = BeautifulSoup(response.text, "lxml")
return soup
def get_reviews(soup):
review_elements = soup.select("div.review")
scraped_reviews = []
for review in review_elements:
r_author_element = review.select_one("span.a-profile-name")
r_author = r_author_element.text if r_author_element else None
r_rating_element = review.select_one("i.review-rating")
r_rating = r_rating_element.text.replace("out of 5 stars", "") if r_rating_element else None
r_title_element = review.select_one("a.review-title")
r_title_span_element = r_title_element.select_one("span:not([class])") if r_title_element else None
r_title = r_title_span_element.text if r_title_span_element else None
r_content_element = review.select_one("span.review-text")
r_content = r_content_element.text if r_content_element else None
r_date_element = review.select_one("span.review-date")
r_date = r_date_element.text if r_date_element else None
r_verified_element = review.select_one("span.a-size-mini")
r_verified = r_verified_element.text if r_verified_element else None
r_image_element = review.select_one("img.review-image-tile")
r_image = r_image_element.attrs["src"] if r_image_element else None
r = {
"author": r_author,
"rating": r_rating,
"title": r_title,
"content": r_content,
"date": r_date,
"verified": r_verified,
"image_url": r_image
}
scraped_reviews.append(r)
return scraped_reviews
def scrape_all_pages(url):
all_reviews = []
page_number = 1
while True:
soup = get_soup(f"{url}&pageNumber={page_number}")
reviews = get_reviews(soup)
if not reviews: # Break the loop if no reviews found on this page
break
all_reviews.extend(reviews)
page_number += 1
return all_reviews
# # Example usage:
# url = "https://www.amazon.in/OnePlus-Nord-Pastel-128GB-Storage/product-reviews/B0BY8JZ22K/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews"
# all_reviews = scrape_all_pages(url)
# # Convert to DataFrame for further analysis
# df = pd.DataFrame(all_reviews)
# df
|