Spaces:
Sleeping
Sleeping
File size: 5,605 Bytes
ce57c5d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
import re
import csv
import json
import time
import random
import requests
import feedparser
from bs4 import BeautifulSoup
from urllib.parse import quote
from datetime import datetime
def extract_data_from_publication_info(publication_info):
authors = ""
journal = ""
year = ""
publisher = ""
regex_pattern = r"(.+?)\s+-\s+(.+?),\s+(\d{4})\s+-\s+(.+)$"
match = re.match(regex_pattern, publication_info)
if match:
authors = match.group(1).strip()
journal = match.group(2).strip()
year = match.group(3).strip()
publisher = match.group(4).strip()
else:
authors = "Unknown"
journal = "Unknown"
year = "Unknown"
publisher = "Unknown"
return {
"authors": authors,
"journal": journal,
"year": year,
"publisher": publisher
}
def scrape_gg_scholar(query, num_pages=1, start_year=None, end_year=None):
results = []
header = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
base_url = "https://scholar.google.com/scholar?"
params = {
"q": query.replace(" ", "+"),
"hl": "en",
"as_sdt": "0,5"
}
if start_year and end_year:
params["as_ylo"] = start_year
params["as_yhi"] = end_year
for pages in range(num_pages):
start = pages * 10
params["start"] = start
url_params = "&".join([f"{k}={v}" for k, v in params.items()])
url = base_url + url_params
try:
response = requests.get(url, headers=header)
if response.status_code == 200:
soup = BeautifulSoup(response.text, "html.parser")
articles = soup.select(".gs_r.gs_or.gs_scl")
for article in articles:
title_element = article.select_one(".gs_rt a")
if title_element:
title = title_element.text
link = title_element["href"]
abstract_element = article.select_one(".gs_rs")
abstract = abstract_element.text if abstract_element else ""
abstract = abstract.replace("…", "").strip()
abstract = abstract.replace("\n", "").strip()
abstract = " ".join(abstract.split())
pub_info_element = article.select_one(".gs_a")
pub_info = pub_info_element.text if pub_info_element else ""
pub_info_parsed = extract_data_from_publication_info(pub_info)
results.append({
"GGS_title": title,
"GGS_link": link,
"GGS_brief_abstract": abstract,
"GGS_publication_info": pub_info,
"GGS_authors": pub_info_parsed["authors"],
"GGS_journal": pub_info_parsed["journal"],
"GGS_year": pub_info_parsed["year"],
"GGS_publisher": pub_info_parsed["publisher"]
})
time.sleep(random.uniform(1, 3)) # Random sleep to avoid being blocked
else:
print(f"ERROR: STATUS CODE {response.status_code}")
break
except Exception as e:
print(f"An error occurred: {e}")
break
return results
def save_to_csv(data, filename="base_crawling_from_gg_scholar.csv"):
with open(filename, "w", newline="", encoding="utf-8-sig") as csvfile:
fieldnames = data[0].keys()
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for row in data:
writer.writerow(row)
print(f"Data saved to {filename}")
def main():
query = input("Enter your search query: ")
num_pages = int(input("Enter the number of pages to scrape: "))
use_time_filter = input("Do you want to filter by year? (y/n): ").strip().lower()
start_year = None
end_year = None
if use_time_filter == 'y':
start_year = input("Enter the start year (format: YYYY; for example: 2020): ")
end_year = input("Enter the end year (format: YYYY; for example: 2025): ")
results = scrape_gg_scholar(query, num_pages, start_year, end_year)
print(f"Found {len(results)} results.")
for i, result in enumerate(results):
print(f"{i + 1}. {result['title']}")
print(f" Link: {result['link']}")
print(f" Brief Abstract: {result['brief_abstract']}")
print(f" Publication Info: {result['publication_info']}")
print(f" Authors: {result['authors']}")
print(f" Journal: {result['journal']}")
print(f" Year: {result['year']}")
print(f" Publisher: {result['publisher']}")
print("=" * 100)
save_option = input("Do you want to save the results to a CSV file? (y/n): ").strip().lower()
if save_option == 'y':
file_name = input("Enter the filename (default file name is 'base_crawling_from_gg_scholar.csv', enter fine name without extension): ")
save_to_csv(results)
if __name__ == "__main__":
main()
# The code is designed to scrape Google Scholar for academic articles based on a search query. |