Spaces:
Sleeping
Sleeping
File size: 4,185 Bytes
60d8d9d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
import pandas as pd
import os
import re
from nltk.corpus import stopwords
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from collections import defaultdict
import spacy
import string
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
# Load the spaCy model for English
nlp = spacy.load("en_core_web_sm")
# Get English stop words from NLTK
stop_words = set(stopwords.words("english"))
def process_text(text):
"""
Process text by:
1. Lowercasing
2. Removing punctuation and non-alphanumeric characters
3. Removing stop words
4. Lemmatization
"""
# Step 1: Tokenization & Processing with spaCy
doc = nlp(text.lower()) # Process text with spaCy
# Step 2: Filter out stop words, non-alphanumeric characters, punctuation, and apply lemmatization
processed_tokens = [
re.sub(r'[^a-zA-Z0-9]', '', token.lemma_) # Remove non-alphanumeric characters
for token in doc
if token.text not in stop_words and token.text not in string.punctuation
]
# Optional: Filter out empty strings resulting from the regex replacement
processed_tokens = [word for word in processed_tokens if word]
return processed_tokens
def generate_word_clouds_by_category(df, output_dir="wordclouds"):
"""
Generates and saves word clouds for each category in the DataFrame.
Args:
df (pd.DataFrame): DataFrame with 'processed_text' and 'category' columns.
output_dir (str): Directory to save the word cloud images.
"""
# Create output directory if it doesn't exist
os.makedirs(output_dir, exist_ok=True)
# Group words by category
category_word_map = defaultdict(list)
for _, row in df.iterrows():
category_word_map[row["label"]].extend(row["processed_quote"])
# Generate and save word clouds
for category, words in category_word_map.items():
word_cloud = WordCloud(width=800, height=400, background_color='white').generate(" ".join(words))
# Plot and save the word cloud
plt.figure(figsize=(10, 5))
plt.imshow(word_cloud, interpolation='bilinear')
plt.axis('off')
plt.title(category)
# Save the plot as an image
filename = os.path.join(output_dir, f"{category.replace(' ', '_').lower()}_wordcloud.png")
plt.savefig(filename, bbox_inches='tight')
print(f"Word cloud saved for category '{category}' at {filename}")
plt.close() # Close the figure to avoid memory issues
def plot_embeddings_2d(df, embedding_column, label_column, method='PCA', random_state=42):
"""
This function reduces high-dimensional embeddings into 2D and visualizes them.
Args:
df (pd.DataFrame): DataFrame containing the text data and labels.
embedding_column (str): The column containing the embeddings.
label_column (str): The column containing the labels.
method (str): The dimensionality reduction method ('PCA' or 'tSNE').
random_state (int): Random state for reproducibility.
"""
# Step 1: Use dimensionality reduction (PCA or t-SNE)
if method == 'PCA':
reducer = PCA(n_components=2, random_state=random_state)
elif method == 'tSNE':
reducer = TSNE(n_components=2, random_state=random_state)
else:
raise ValueError("Invalid method. Use 'PCA' or 'tSNE'.")
# Reduce the embeddings to 2D
embeddings_2d = reducer.fit_transform(df[embedding_column].tolist())
# Step 2: Plot the 2D embeddings
plt.figure(figsize=(10, 8))
# Scatter plot, coloring points by their label
scatter = plt.scatter(embeddings_2d[:, 0], embeddings_2d[:, 1], c=df[label_column], cmap='viridis')
# Create a legend for the classes (labels)
plt.legend(*scatter.legend_elements(), title="Classes")
# Adding labels and title
plt.title("2D Visualization of Embeddings")
plt.xlabel("Principal Component 1" if method == 'PCA' else "t-SNE Dimension 1")
plt.ylabel("Principal Component 2" if method == 'PCA' else "t-SNE Dimension 2")
plt.colorbar(scatter)
plt.show() |