Update app.py

app.py

@@ -10,6 +10,7 @@ import os
 HF_TOKEN = os.getenv('HF_TOKEN')
 
 if HF_TOKEN:
+    HF_TOKEN = HF_TOKEN.strip()  # Remove any leading or trailing whitespace/newlines
     login(token=HF_TOKEN)
 
 # Load additional tokenizers from transformers
@@ -63,29 +64,16 @@ if meta_llama_tokenizer:
     tokenizers["meta-llama/Meta-Llama-3-8B"] = lambda: meta_llama_tokenizer
 
 def compare_tokenizers(tokenizer_name, text):
-    # Handle the transformer tokenizers separately due to API differences
-    if tokenizer_name in [
-        "FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", 
-        "inception-mbzuai/jais-13b", "aubmindlab/bert-base-arabertv2",
-        "meta-llama/Meta-Llama-3-8B", "CohereForAI/c4ai-command-r-v01", "CohereForAI/c4ai-command-r-plus"
-    ]:
-        tokenizer = tokenizers[tokenizer_name]()
-        tokens = tokenizer.tokenize(text)
-        encoded_output = tokenizer.encode(text, add_special_tokens=True)
-        decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
-    else:
-        # AraNizer tokenizers
-        tokenizer = tokenizers[tokenizer_name]()
-        tokens = tokenizer.tokenize(text)
-        encoded_output = tokenizer.encode(text, add_special_tokens=True)
-        decoded_text = tokenizer.decode(encoded_output)
-    
+    tokenizer = tokenizers[tokenizer_name]()
+    tokens = tokenizer.tokenize(text)
+    encoded_output = tokenizer.encode(text, add_special_tokens=True)
+    decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
+
     # Prepare the results to be displayed in HTML format
-    tokens_arabic = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
     results_html = f"""
     <div>
         <h3>Tokenizer: {tokenizer_name}</h3>
-        <p><strong>Tokens:</strong> {tokens_arabic}</p>
+        <p><strong>Tokens:</strong> {tokens}</p>
         <p><strong>Encoded:</strong> {encoded_output}</p>
         <p><strong>Decoded:</strong> {decoded_text}</p>
     </div>