add tonic tokenizer

tasks/text.py

@@ -60,17 +60,22 @@ async def evaluate_text(request: TextEvaluationRequest):
         
         # Model and tokenizer paths
         path_model = 'Tonic/climate-guard-toxic-agent'
-        path_tokenizer = "answerdotai/ModernBERT-base"
+        path_tokenizer = "Tonic/climate-guard-toxic-agent"
         
-        # Load tokenizer and model
+        # Load tokenizer
         tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
+        
+        # Load model
         model = AutoModelForSequenceClassification.from_pretrained(
             path_model,
             trust_remote_code=True,
-            torch_dtype=torch.float16
+            num_labels=8,
+            problem_type="single_label_classification",
+            ignore_mismatched_sizes=True
         ).to(device)
         
-        # Set model to evaluation mode
+        # Convert to half precision and eval mode
+        model = model.half()
         model.eval()
         
         # Preprocess function
@@ -78,7 +83,6 @@ async def evaluate_text(request: TextEvaluationRequest):
             return tokenizer(
                 examples["quote"],
                 truncation=True,
-                padding=True,
                 return_tensors=None
             )
         
@@ -124,6 +128,7 @@ async def evaluate_text(request: TextEvaluationRequest):
         print(f"Error during model inference: {str(e)}")
         raise
 
+
     #--------------------------------------------------------------------------------------------
     # MODEL INFERENCE ENDS HERE
     #--------------------------------------------------------------------------------------------