Update app.py

app.py

@@ -1,79 +1,63 @@
 from gradio import Interface
 import gradio as gr
 import aranizer
-from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 from transformers import AutoTokenizer
 import codecs
 
-# Load additional tokenizers from transformers
+# Loading tokenizer instances from Transformers.
 gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
 gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
 jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
 
-# List of available tokenizers and a dictionary to load them
-tokenizer_options = [
-    "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
-    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
-    "FreedomIntelligence/AceGPT-13B",  # Previously added GPT tokenizer
-    "FreedomIntelligence/AceGPT-7B",   # Another previously added GPT tokenizer
-    "inception-mbzuai/jais-13b"        # Adding the new tokenizer to the options
-]
-
+# Assuming the existence of get_tokenizer() method for aranizer models in your setup.
 tokenizers = {
-    "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
-    "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
-    "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer,
-    "aranizer_sp32k": aranizer_sp32k.get_tokenizer,
-    "aranizer_sp50k": aranizer_sp50k.get_tokenizer,
-    "aranizer_sp64k": aranizer_sp64k.get_tokenizer,
-    "aranizer_sp86k": aranizer_sp86k.get_tokenizer,
+    "aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
+    "aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
+    "aranizer_bpe86k": lambda: aranizer.aranizer_bpe86k.get_tokenizer(),
+    "aranizer_sp32k": lambda: aranizer.aranizer_sp32k.get_tokenizer(),
+    "aranizer_sp50k": lambda: aranizer.aranizer_sp50k.get_tokenizer(),
+    "aranizer_sp64k": lambda: aranizer.aranizer_sp64k.get_tokenizer(),
+    "aranizer_sp86k": lambda: aranizer.aranizer_sp86k.get_tokenizer(),
     "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
-    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer  # Adding the new Jais tokenizer
+    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
 }
 
+# Define tokenizer options for dropdown menu.
+tokenizer_options = list(tokenizers.keys())
+
 def compare_tokenizers(tokenizer_name, text):
-    # Handle the transformer tokenizers separately due to API differences
-    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b"]:
-        tokenizer = tokenizers[tokenizer_name]()
-        tokens = tokenizer.tokenize(text)
-        encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
-        decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
-        # Decoding using codecs to force UTF-8 and handle potential encoding issues
-        decoded_text_utf8 = codecs.decode(decoded_text.encode('utf-8'), 'utf-8', errors='ignore')
-    else:
-        # AraNizer tokenizers
-        tokenizer = tokenizers[tokenizer_name]()
-        tokens = tokenizer.tokenize(text)
-        encoded_output = tokenizer.encode(text, add_special_tokens=True)
-        decoded_text = tokenizer.decode(encoded_output)
-        # Same codecs handling for AraNizer tokenizers
-        decoded_text_utf8 = codecs.decode(decoded_text.encode('utf-8'), 'utf-8', errors='ignore')
+    # UTF-8 encoding assertion for the input text
+    text = codecs.decode(text.encode('utf-8'), 'utf-8')
     
-    # Prepare the results to be displayed, using UTF-8 decoded text
-    results = [(tokenizer_name, tokens, encoded_output, decoded_text_utf8)]
+    tokenizer = tokenizers[tokenizer_name]()
+    tokens = tokenizer.tokenize(text)
+    encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
+    decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
+    
+    # Ensuring the tokens are iterated and converted correctly
+    tokens_utf8 = [codecs.decode(token.encode('utf-8'), 'utf-8', errors='ignore') for token in tokens]
+
+    # Preparing and returning results in UTF-8
+    results = [(tokenizer_name, tokens_utf8, encoded_output.tolist(), decoded_text)]
     return results
 
-# Define the Gradio interface components with a dropdown for model selection
 inputs_component = [
     gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
-    gr.Textbox(lines=2, placeholder="Enter text here...", label="Input Text")
+    gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text", default="مثال بالعربية")
 ]
 
 outputs_component = gr.Dataframe(
     headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"], 
     label="Results",
-    type="pandas"
 )
 
-# Setting up the interface
 iface = Interface(
     fn=compare_tokenizers, 
     inputs=inputs_component, 
     outputs=outputs_component, 
     title="Tokenizer Comparison",
-    live=True
+    live=True,
 )
 
-# Launching the Gradio app
 iface.launch()