File size: 4,445 Bytes
d187b57 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
import pandas as pd
import numpy as np
from text_preprocessor import TextPreprocessor
from tqdm import tqdm
import logging
from pathlib import Path
import time
def process_dataset(input_path: str, output_path: str = None, batch_size: int = 1000):
"""
Process a dataset using the TextPreprocessor with efficient batch processing.
Args:
input_path: Path to input CSV file
output_path: Path to save processed CSV file. If None, will use input name with _processed suffix
batch_size: Number of texts to process in each batch
"""
# Setup output path
if output_path is None:
input_path = Path(input_path)
output_path = input_path.parent / f"{input_path.stem}_processed{input_path.suffix}"
# Initialize preprocessor
preprocessor = TextPreprocessor()
print(f"\nProcessing dataset: {input_path}")
start_time = time.time()
try:
# Read the dataset
print("Reading dataset...")
df = pd.read_csv(input_path)
total_rows = len(df)
print(f"Total rows: {total_rows:,}")
# Process in batches with progress bar
print("\nProcessing text...")
# Calculate number of batches
num_batches = (total_rows + batch_size - 1) // batch_size
for i in tqdm(range(0, total_rows, batch_size), total=num_batches, desc="Processing batches"):
# Get batch
batch_start = i
batch_end = min(i + batch_size, total_rows)
# Process each text in the batch
for idx in range(batch_start, batch_end):
text = df.loc[idx, 'comment_text']
lang = df.loc[idx, 'lang'] if 'lang' in df.columns else 'en'
# Process text
processed = preprocessor.preprocess_text(
text,
lang=lang,
clean_options={
'remove_stops': True,
'remove_numbers': True,
'remove_urls': True,
'remove_emails': True,
'remove_mentions': True,
'remove_hashtags': True,
'expand_contractions': True,
'remove_accents': False,
'min_word_length': 2
},
do_stemming=True
)
# Update the text directly
df.loc[idx, 'comment_text'] = processed
# Optional: Print sample from first batch
if i == 0:
print("\nSample processing results:")
for j in range(min(3, batch_size)):
print(f"\nProcessed text {j+1}: {df.loc[j, 'comment_text'][:100]}...")
# Save processed dataset
print(f"\nSaving processed dataset to: {output_path}")
df.to_csv(output_path, index=False)
# Print statistics
end_time = time.time()
processing_time = end_time - start_time
print("\nProcessing Complete!")
print("-" * 50)
print(f"Total rows processed: {total_rows:,}")
print(f"Processing time: {processing_time/60:.2f} minutes")
print(f"Average time per text: {processing_time/total_rows*1000:.2f} ms")
print(f"Output file size: {Path(output_path).stat().st_size/1024/1024:.1f} MB")
# Print sample of unique words before and after
print("\nVocabulary Statistics:")
sample_size = min(1000, total_rows)
original_words = set(' '.join(df['comment_text'].head(sample_size).astype(str)).split())
processed_words = set(' '.join(df['processed_text'].head(sample_size).astype(str)).split())
print(f"Sample unique words (first {sample_size:,} rows):")
print(f"Before processing: {len(original_words):,}")
print(f"After processing : {len(processed_words):,}")
print(f"Reduction: {(1 - len(processed_words)/len(original_words))*100:.1f}%")
except Exception as e:
print(f"\nError processing dataset: {str(e)}")
raise
if __name__ == "__main__":
# Process training dataset
input_file = "dataset/split/train.csv"
output_file = "dataset/split/train_no_stopwords.csv"
process_dataset(input_file, output_file) |