Update app.py

app.py

@@ -207,54 +207,63 @@ def load_model():
             log.append(f"Alternative loading also failed: {e2}")
             return "\n".join(log)
     
-    # Try to load the tokenizer from the model repository directly
+    # --- Load Tokenizer ---
     progress(0.3, desc="Loading tokenizer...")
     try:
-        # First attempt: Try loading from local path
-        tokenizer = AutoTokenizer.from_pretrained(
-            local_model_path,
-            padding_side="right",
-            use_fast=True,
-        )
-        log.append("Tokenizer loaded from local files")
-    except Exception as e:
-        log.append(f"Could not load tokenizer from local files: {e}")
+        log.append("Loading a compatible tokenizer...")
+        # Use the tokenizer from Meta's official Llama models - should be compatible with Llama 3.2
+        tokenizer_id = "meta-llama/Llama-3-8B"  # This is a reliable source for a Llama tokenizer
         
-        # Second attempt: Try loading directly from HF repo
+        # Try with specified tokenizer first
         try:
-            log.append("Attempting to load tokenizer directly from Hugging Face...")
             tokenizer = AutoTokenizer.from_pretrained(
-                hf_model_repo_id,
-                padding_side="right",
+                tokenizer_id,
                 use_fast=True,
+                padding_side="right",
+                trust_remote_code=True
             )
-            log.append("Tokenizer loaded from Hugging Face repository")
-        except Exception as e2:
-            # Third attempt: Try loading a compatible tokenizer
-            log.append(f"Could not load tokenizer from repo: {e2}")
-            log.append("Attempting to load a compatible LlamaTokenizer...")
+            log.append(f"Successfully loaded tokenizer from {tokenizer_id}")
+        except Exception as e:
+            log.append(f"Could not load from {tokenizer_id}: {e}")
+            # Fallback to Llama-2 tokenizer
             try:
+                tokenizer = AutoTokenizer.from_pretrained(
+                    "meta-llama/Llama-2-7b-hf",
+                    use_fast=True,
+                    padding_side="right"
+                )
+                log.append("Loaded Llama-2 tokenizer as fallback")
+            except Exception as e2:
+                # If that fails too, try the most basic option
                 from transformers import LlamaTokenizer
-                
-                # Try Meta's standard Llama tokenizer
                 tokenizer = LlamaTokenizer.from_pretrained(
-                    "meta-llama/Llama-2-7b-hf",  # Standard Llama tokenizer
-                    padding_side="right",
-                    use_fast=False,  # Try the Python version
+                    "hf-internal-testing/llama-tokenizer",
+                    use_fast=False,
+                    padding_side="right"
                 )
-                log.append("Loaded a compatible LlamaTokenizer as fallback")
-            except Exception as e3:
-                error_msg = f"Failed to load any compatible tokenizer: {e3}"
-                log.append(error_msg)
-                return "\n".join(log)
+                log.append("Loaded basic Llama tokenizer from testing repo")
+            
+        # Set pad token if not already set
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+            log.append("Set pad_token to eos_token")
+        
+        # Make sure we have necessary special tokens
+        if tokenizer.bos_token is None:
+            tokenizer.bos_token = "<s>"
+            log.append("Set bos_token to <s>")
+        
+        if tokenizer.eos_token is None:
+            tokenizer.eos_token = "</s>"
+            log.append("Set eos_token to </s>")
+        
+        log.append(f"Loaded model vocab size: {len(tokenizer)}")
+        
+    except Exception as e:
+        error_msg = f"All attempts to load a tokenizer failed: {e}"
+        log.append(error_msg)
+        return "\n".join(log)
 
-    # Set pad token if not already set
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-        log.append("Set pad_token to eos_token")
-    
-    print(f"Loaded tokenizer vocabulary size: {len(tokenizer)}")
-    
     # Print information about input embeddings
     print(f"Input embedding shape: {model.get_input_embeddings().weight.shape}")