Update app.py

app.py

@@ -1,7 +1,10 @@
 from gradio import Interface
 import gradio as gr
 import aranizer
-from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
+from aranizer import (
+    aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k,
+    aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
+)
 from transformers import AutoTokenizer, logging
 from huggingface_hub import login
 import os
@@ -26,19 +29,14 @@ except Exception as e:
     meta_llama_tokenizer = None
     logging.warning(f"Could not load meta-llama/Meta-Llama-3-8B tokenizer: {e}")
 
-cohere_command_r_v01_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")
-cohere_command_r_plus_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-plus")
-
 # List of available tokenizers and a dictionary to load them
 tokenizer_options = [
     "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
     "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
-    "FreedomIntelligence/AceGPT-13B",  
-    "FreedomIntelligence/AceGPT-7B",   
+    "FreedomIntelligence/AceGPT-13B",
+    "FreedomIntelligence/AceGPT-7B",
     "inception-mbzuai/jais-13b",
-    "aubmindlab/bert-base-arabertv2",
-    "CohereForAI/c4ai-command-r-v01",
-    "CohereForAI/c4ai-command-r-plus"
+    "aubmindlab/bert-base-arabertv2"
 ]
 
 if meta_llama_tokenizer:
@@ -55,9 +53,7 @@ tokenizers = {
     "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
     "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
-    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer,
-    "CohereForAI/c4ai-command-r-v01": lambda: cohere_command_r_v01_tokenizer,
-    "CohereForAI/c4ai-command-r-plus": lambda: cohere_command_r_plus_tokenizer
+    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer
 }
 
 if meta_llama_tokenizer:
@@ -112,4 +108,4 @@ iface = Interface(
 )
 
 # Launching the Gradio app
-iface.launch()
+iface.launch()