Update app.py

app.py

@@ -1,13 +1,22 @@
 from flask import Flask, request, render_template
 import pandas as pd
-import spacy-transformers
+import nltk
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
 from transformers import pipeline
 
 # Initialize Flask app
 app = Flask(__name__)
 
-# Load spaCy model for preprocessing
-nlp = spacy-transformers.load("en_core_web_sm")
+# Download NLTK resources
+nltk.download('punkt')
+nltk.download('stopwords')
+nltk.download('wordnet')
+
+# Initialize NLTK components
+lemmatizer = WordNetLemmatizer()
+stop_words = set(stopwords.words('english'))
 
 # Load Hugging Face pipelines
 sentiment_pipeline = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
@@ -15,9 +24,11 @@ ner_pipeline = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-e
 
 # Function to preprocess text
 def preprocess_text(text):
-    doc = nlp(text)
-    tokens = [token.lemma_.lower() for token in doc if not token.is_stop and not token.is_punct]
-    return ' '.join(tokens)
+    # Tokenize
+    tokens = word_tokenize(text)
+    # Remove stop words and lemmatize
+    cleaned_tokens = [lemmatizer.lemmatize(token.lower()) for token in tokens if token.isalpha() and token.lower() not in stop_words]
+    return ' '.join(cleaned_tokens)
 
 @app.route('/')
 def home():