Spaces:
Sleeping
Sleeping
File size: 4,830 Bytes
3c77d98 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
import requests
import pandas as pd
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime
def extract_div_contents_from_url(url):
response = requests.get(url)
if response.status_code != 200:
return pd.DataFrame(columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
soup = BeautifulSoup(response.content, 'html.parser')
divs = soup.find_all('div', class_='boilerplate afd vfd xfd-closed archived mw-archivedtalk')
# Extract the title fragment from the URL
url_fragment = url.split('#')[-1].replace('_', ' ')
data = []
for div in divs:
try:
title = None
text_url = None
# Extract title and text_url
title_tag = div.find('a')
if title_tag:
title_span = div.find('span', {'data-mw-comment-start': True})
if title_span:
title_anchor = title_span.find_next_sibling('a')
if title_anchor:
title = title_anchor.text
text_url = 'https://en.wikipedia.org' + title_anchor['href']
else:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if title == 'talk page' or title is None:
heading_tag = div.find('div', class_='mw-heading mw-heading3')
if heading_tag:
title_tag = heading_tag.find('a')
if title_tag:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if title != url_fragment:
continue # Skip if the title does not match the URL fragment
deletion_discussion = div.prettify()
# Extract label
label = ''
verdict_tag = div.find('p')
if verdict_tag:
label_b_tag = verdict_tag.find('b')
if label_b_tag:
label = label_b_tag.text.strip()
# Extract confirmation
confirmation = ''
discussion_tag = div.find('dd').find('i')
if discussion_tag:
confirmation_b_tag = discussion_tag.find('b')
if confirmation_b_tag:
confirmation = confirmation_b_tag.text.strip()
# Split deletion_discussion into discussion and verdict
parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
discussion = parts[0] if len(parts) > 0 else ''
verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
data.append([title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
except Exception as e:
print(f"Error processing div: {e}")
continue
df = pd.DataFrame(data, columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
df = df[['title','discussion','verdict','label']]
return df
def extract_post_links_text(discussion_html):
split_point = '<span class="plainlinks">'
if split_point in discussion_html:
parts = discussion_html.split(split_point)
if len(parts) > 1:
return parts[1]
return discussion_html
def process_discussion(df):
df['discussion_cleaned'] = df['verdict'].apply(extract_post_links_text)
return df
def html_to_plaintext(html_content):
soup = BeautifulSoup(html_content, 'html.parser')
for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
tag.insert_before('\n')
tag.insert_after('\n')
for br in soup.find_all('br'):
br.replace_with('\n')
text = soup.get_text(separator=' ', strip=True)
text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
return text
def process_html_to_plaintext(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(html_to_plaintext)
df = df[['title', 'discussion_cleaned', 'label']]
return df
import pysbd
def split_text_into_sentences(text):
seg = pysbd.Segmenter(language="en", clean=False)
sentences = seg.segment(text)
return ' '.join(sentences[1:])
def process_split_text_into_sentences(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
return df
def process_data(url):
df = extract_div_contents_from_url(url)
df = process_discussion(df)
df = process_html_to_plaintext(df)
df = process_split_text_into_sentences(df)
#if not empty
if not df.empty:
return df.at[0,'title']+ ' : '+df.at[0, 'discussion_cleaned']
else:
return 'Empty DataFrame' |