fix model loading error

tasks/text.py

@@ -7,7 +7,7 @@ import os
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Dict, Tuple
 import torch
-from transformers import pipeline
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig
 from huggingface_hub import login
 from dotenv import load_dotenv
 
@@ -38,13 +38,26 @@ class TextClassifier:
         
         for attempt in range(max_retries):
             try:
-                # Initialize pipeline
-                self.classifier = pipeline(
-                    "text-classification",
-                    model=model_name,
-                    device=self.device,
-                    batch_size=32
+                # Load config and modify it to remove bias parameter
+                self.config = AutoConfig.from_pretrained(model_name)
+                if hasattr(self.config, 'norm_bias'):
+                    delattr(self.config, 'norm_bias')
+                
+                # Initialize tokenizer
+                self.tokenizer = AutoTokenizer.from_pretrained(
+                    model_name,
+                    model_max_length=2048
+                )
+                
+                # Initialize model with modified config
+                self.model = AutoModelForSequenceClassification.from_pretrained(
+                    model_name,
+                    config=self.config,
+                    ignore_mismatched_sizes=True
                 )
+                
+                self.model.to(self.device)
+                self.model.eval()
                 print("Model initialized successfully")
                 break
                 
@@ -59,9 +72,19 @@ class TextClassifier:
         try:
             print(f"Processing batch {batch_idx} with {len(batch)} items")
             
-            # Use pipeline for prediction
-            results = self.classifier(batch)
-            predictions = [int(result['label'].split('_')[0]) for result in results]
+            # Tokenize
+            inputs = self.tokenizer(
+                batch,
+                padding=True,
+                truncation=True,
+                max_length=2048,
+                return_tensors="pt"
+            ).to(self.device)
+            
+            # Get predictions
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+                predictions = torch.argmax(outputs.logits, dim=-1).cpu().tolist()
             
             print(f"Completed batch {batch_idx} with {len(predictions)} predictions")
             return predictions, batch_idx