Update app.py

app.py

@@ -207,16 +207,51 @@ def load_model():
             log.append(f"Alternative loading also failed: {e2}")
             return "\n".join(log)
     
-    # Load the official Meta tokenizer for LLaMA 3
-    tokenizer = AutoTokenizer.from_pretrained(
-        "meta-llama/Llama-3-8B",  # Use the official Meta tokenizer
-        use_auth_token=os.environ.get("HF_TOKEN", None)  # In case it's needed
-    )
-    
-    if tokenizer is None:
-        # Fallback to another common foundation model tokenizer
-        print("Falling back to another tokenizer as Meta tokenizer requires auth token")
-        tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
+    # Try to load the tokenizer from the model repository directly
+    progress(0.3, desc="Loading tokenizer...")
+    try:
+        # First attempt: Try loading from local path
+        tokenizer = AutoTokenizer.from_pretrained(
+            local_model_path,
+            padding_side="right",
+            use_fast=True,
+        )
+        log.append("Tokenizer loaded from local files")
+    except Exception as e:
+        log.append(f"Could not load tokenizer from local files: {e}")
+        
+        # Second attempt: Try loading directly from HF repo
+        try:
+            log.append("Attempting to load tokenizer directly from Hugging Face...")
+            tokenizer = AutoTokenizer.from_pretrained(
+                hf_model_repo_id,
+                padding_side="right",
+                use_fast=True,
+            )
+            log.append("Tokenizer loaded from Hugging Face repository")
+        except Exception as e2:
+            # Third attempt: Try loading a compatible tokenizer
+            log.append(f"Could not load tokenizer from repo: {e2}")
+            log.append("Attempting to load a compatible LlamaTokenizer...")
+            try:
+                from transformers import LlamaTokenizer
+                
+                # Try Meta's standard Llama tokenizer
+                tokenizer = LlamaTokenizer.from_pretrained(
+                    "meta-llama/Llama-2-7b-hf",  # Standard Llama tokenizer
+                    padding_side="right",
+                    use_fast=False,  # Try the Python version
+                )
+                log.append("Loaded a compatible LlamaTokenizer as fallback")
+            except Exception as e3:
+                error_msg = f"Failed to load any compatible tokenizer: {e3}"
+                log.append(error_msg)
+                return "\n".join(log)
+
+    # Set pad token if not already set
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+        log.append("Set pad_token to eos_token")
     
     print(f"Loaded tokenizer vocabulary size: {len(tokenizer)}")