Update app.py

app.py

@@ -0,0 +1,426 @@
+import gradio as gr
+import os
+import threading
+import time
+from pathlib import Path
+from huggingface_hub import login
+
+
+# Try to import llama-cpp-python, fallback to instructions if not available
+try:
+    from llama_cpp import Llama
+    LLAMA_CPP_AVAILABLE = True
+except ImportError:
+    LLAMA_CPP_AVAILABLE = False
+    print("llama-cpp-python not installed. Please install it with: pip install llama-cpp-python")
+
+hf_token = os.environ.get("HF_TOKEN")
+
+login(token = hf_token)
+
+
+# Global variables for model
+model = None
+model_loaded = False
+
+def find_gguf_file(directory="."):
+    """Find GGUF files in the specified directory"""
+    gguf_files = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.gguf'):
+                gguf_files.append(os.path.join(root, file))
+    return gguf_files
+
+def get_optimal_settings():
+    """Get optimal CPU threads and GPU layers automatically"""
+    # Auto-detect CPU threads (use all available cores)
+    n_threads = os.cpu_count()
+    
+    # Auto-detect GPU layers (try to use GPU if available)
+    n_gpu_layers = 0
+    try:
+        # Try to detect if CUDA is available
+        import subprocess
+        result = subprocess.run(['nvidia-smi'], capture_output=True, text=True)
+        if result.returncode == 0:
+            # NVIDIA GPU detected, use more layers
+            n_gpu_layers = 35  # Good default for Llama-3-8B
+    except:
+        # No GPU or CUDA not available
+        n_gpu_layers = 0
+    
+    return n_threads, n_gpu_layers
+
+def load_model_from_huggingface(repo_id, filename, n_ctx=2048):
+    """Load the model from Hugging Face repository"""
+    global model, model_loaded
+    
+    if not LLAMA_CPP_AVAILABLE:
+        return False, "llama-cpp-python not installed. Please install it with: pip install llama-cpp-python"
+    
+    try:
+        print(f"Loading model from Hugging Face: {repo_id}/{filename}")
+        
+        # Get optimal settings automatically
+        n_threads, n_gpu_layers = get_optimal_settings()
+        print(f"Auto-detected settings: {n_threads} CPU threads, {n_gpu_layers} GPU layers")
+        
+        # Load model from Hugging Face with optimized settings
+        model = Llama.from_pretrained(
+            repo_id=repo_id,
+            filename=filename,
+            n_ctx=n_ctx,  # Context window (configurable)
+            n_threads=n_threads,  # CPU threads (auto-detected)
+            n_gpu_layers=n_gpu_layers,  # Number of layers to offload to GPU (auto-detected)
+            verbose=False,
+            chat_format="chatml",  # Use Llama-3 chat format
+            n_batch=512,  # Batch size for prompt processing
+            use_mlock=True,  # Keep model in memory
+            use_mmap=True,  # Use memory mapping
+        )
+        
+        model_loaded = True
+        print("Model loaded successfully!")
+        return True, f"✅ Model loaded successfully from {repo_id}/{filename}\n📊 Context: {n_ctx} tokens\n🖥️ CPU Threads: {n_threads}\n🎮 GPU Layers: {n_gpu_layers}"
+        
+    except Exception as e:
+        model_loaded = False
+        error_msg = f"Error loading model: {str(e)}"
+        print(error_msg)
+        return False, f"❌ {error_msg}"
+
+def load_model_from_gguf(gguf_path=None, n_ctx=2048):
+    """Load the model from a local GGUF file with automatic optimization"""
+    global model, model_loaded
+    
+    if not LLAMA_CPP_AVAILABLE:
+        return False, "llama-cpp-python not installed. Please install it with: pip install llama-cpp-python"
+    
+    try:
+        # If no path provided, try to find GGUF files
+        if gguf_path is None:
+            gguf_files = find_gguf_file()
+            if not gguf_files:
+                return False, "No GGUF files found in the repository"
+            gguf_path = gguf_files[0]  # Use the first one found
+            print(f"Found GGUF file: {gguf_path}")
+        
+        # Check if file exists
+        if not os.path.exists(gguf_path):
+            return False, f"GGUF file not found: {gguf_path}"
+        
+        print(f"Loading model from: {gguf_path}")
+        
+        # Get optimal settings automatically
+        n_threads, n_gpu_layers = get_optimal_settings()
+        print(f"Auto-detected settings: {n_threads} CPU threads, {n_gpu_layers} GPU layers")
+        
+        # Load model with optimized settings
+        model = Llama(
+            model_path=gguf_path,
+            n_ctx=n_ctx,  # Context window (configurable)
+            n_threads=n_threads,  # CPU threads (auto-detected)
+            n_gpu_layers=n_gpu_layers,  # Number of layers to offload to GPU (auto-detected)
+            verbose=False,
+            chat_format="llama-3",  # Use Llama-3 chat format
+            n_batch=512,  # Batch size for prompt processing
+            use_mlock=True,  # Keep model in memory
+            use_mmap=True,  # Use memory mapping
+        )
+        
+        model_loaded = True
+        print("Model loaded successfully!")
+        return True, f"✅ Model loaded successfully from {os.path.basename(gguf_path)}\n📊 Context: {n_ctx} tokens\n🖥️ CPU Threads: {n_threads}\n🎮 GPU Layers: {n_gpu_layers}"
+        
+    except Exception as e:
+        model_loaded = False
+        error_msg = f"Error loading model: {str(e)}"
+        print(error_msg)
+        return False, f"❌ {error_msg}"
+
+def generate_response_stream(message, history, max_tokens=512, temperature=0.7, top_p=0.9, repeat_penalty=1.1):
+    """Generate response from the model with streaming"""
+    global model, model_loaded
+    
+    if not model_loaded or model is None:
+        yield "Error: Model not loaded. Please load the model first."
+        return
+    
+    try:
+        # Format the conversation history for Llama-3
+        conversation = []
+        
+        # Add conversation history
+        for human, assistant in history:
+            conversation.append({"role": "user", "content": human})
+            if assistant:  # Only add if assistant response exists
+                conversation.append({"role": "assistant", "content": assistant})
+        
+        # Add current message
+        conversation.append({"role": "user", "content": message})
+        
+        # Generate response with streaming
+        response = ""
+        stream = model.create_chat_completion(
+            messages=conversation,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            repeat_penalty=repeat_penalty,
+            stream=True,
+            stop=["<|eot_id|>", "<|end_of_text|>"]
+        )
+        
+        for chunk in stream:
+            if chunk['choices'][0]['delta'].get('content'):
+                new_text = chunk['choices'][0]['delta']['content']
+                response += new_text
+                yield response
+                
+    except Exception as e:
+        yield f"Error generating response: {str(e)}"
+
+def chat_interface(message, history, max_tokens, temperature, top_p, repeat_penalty):
+    """Main chat interface function"""
+    if not message.strip():
+        return history, ""
+    
+    if not model_loaded:
+        history.append((message, "Please load the model first using the 'Load Model' button."))
+        return history, ""
+    
+    # Add user message to history
+    history = history + [(message, "")]
+    
+    # Generate response
+    for response in generate_response_stream(message, history[:-1], max_tokens, temperature, top_p, repeat_penalty):
+        history[-1] = (message, response)
+        yield history, ""
+
+def clear_chat():
+    """Clear the chat history"""
+    return [], ""
+
+def load_model_interface(source_type, gguf_path, repo_id, filename, context_size):
+    """Interface function to load model with configurable context size"""
+    if source_type == "Hugging Face":
+        success, message = load_model_from_huggingface(repo_id, filename, n_ctx=int(context_size))
+    else:  # Local file
+        success, message = load_model_from_gguf(gguf_path, n_ctx=int(context_size))
+    return message
+
+def get_available_gguf_files():
+    """Get list of available GGUF files"""
+    gguf_files = find_gguf_file()
+    if not gguf_files:
+        return ["No GGUF files found"]
+    return [os.path.basename(f) for f in gguf_files]
+
+# Create the Gradio interface
+def create_interface():
+    # Get available GGUF files
+    gguf_files = find_gguf_file()
+    gguf_choices = [os.path.basename(f) for f in gguf_files] if gguf_files else ["No GGUF files found"]
+    
+    with gr.Blocks(title="Llama-3-8B GGUF Chatbot", theme=gr.themes.Soft()) as demo:
+        gr.HTML("""
+            <h1 style="text-align: center; color: #2E86AB; margin-bottom: 30px;">
+                🦙 MMed-Llama-Alpaca GGUF Chatbot
+            </h1>
+            <p style="text-align: center; color: #666; margin-bottom: 30px;">
+                Chat with the MMed-Llama-Alpaca model (Q4_K_M quantized) for medical assistance!
+            </p>
+        """)
+        
+        with gr.Row():
+            with gr.Column(scale=4):
+                # Chat interface
+                chatbot = gr.Chatbot(
+                    height=500,
+                    show_copy_button=True,
+                    bubble_full_width=False,
+                    show_label=False,
+                    placeholder="Model not loaded. Please load the model first to start chatting."
+                )
+                
+                with gr.Row():
+                    msg = gr.Textbox(
+                        placeholder="Type your message here...",
+                        container=False,
+                        scale=7,
+                        show_label=False
+                    )
+                    submit_btn = gr.Button("Send", variant="primary", scale=1)
+                    clear_btn = gr.Button("Clear", variant="secondary", scale=1)
+                
+            with gr.Column(scale=1):
+                # Model loading section
+                gr.HTML("<h3>🔧 Model Control</h3>")
+                
+                # Model source selection
+                source_type = gr.Radio(
+                    choices=["Hugging Face", "Local File"],
+                    value="Hugging Face",
+                    label="Model Source",
+                    info="Choose where to load the model from"
+                )
+                
+                # Hugging Face settings
+                with gr.Group(visible=True) as hf_group:
+                    gr.HTML("<h4>🤗 Hugging Face Settings</h4>")
+                    repo_id = gr.Textbox(
+                        value="Axcel1/MMed-llama-alpaca-Q4_K_M-GGUF",
+                        label="Repository ID",
+                        info="e.g., username/repo-name"
+                    )
+                    filename = gr.Textbox(
+                        value="mmed-llama-alpaca-q4_k_m.gguf",
+                        label="Filename",
+                        info="GGUF filename in the repository"
+                    )
+                
+                # Local file settings
+                with gr.Group(visible=False) as local_group:
+                    gr.HTML("<h4>📁 Local File Settings</h4>")
+                    if gguf_files:
+                        gguf_dropdown = gr.Dropdown(
+                            choices=gguf_choices,
+                            value=gguf_choices[0] if gguf_choices[0] != "No GGUF files found" else None,
+                            label="Select GGUF File",
+                            info="Choose which GGUF file to load"
+                        )
+                    else:
+                        gguf_dropdown = gr.Textbox(
+                            value="No GGUF files found in repository",
+                            label="GGUF File",
+                            interactive=False
+                        )
+                
+                load_btn = gr.Button("Load Model", variant="primary", size="lg")
+                model_status = gr.Textbox(
+                    label="Status",
+                    value="Model not loaded. Configure settings and click 'Load Model'.\n⚙️ Auto-optimized: CPU threads & GPU layers auto-detected\n📝 Context size can be configured in Generation Settings",
+                    interactive=False,
+                    max_lines=5
+                )
+                
+                # Generation parameters
+                gr.HTML("<h3>⚙️ Generation Settings</h3>")
+                
+                # Context size (now as a slider)
+                context_size = gr.Slider(
+                    minimum=512,
+                    maximum=8192,
+                    value=2048,
+                    step=256,
+                    label="Context Size",
+                    info="Token context window (requires model reload)"
+                )
+                
+                max_tokens = gr.Slider(
+                    minimum=50,
+                    maximum=2048,
+                    value=512,
+                    step=50,
+                    label="Max Tokens",
+                    info="Maximum response length"
+                )
+                temperature = gr.Slider(
+                    minimum=0.1,
+                    maximum=2.0,
+                    value=0.7,
+                    step=0.1,
+                    label="Temperature",
+                    info="Creativity (higher = more creative)"
+                )
+                top_p = gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.9,
+                    step=0.1,
+                    label="Top-p",
+                    info="Nucleus sampling"
+                )
+                repeat_penalty = gr.Slider(
+                    minimum=1.0,
+                    maximum=1.5,
+                    value=1.1,
+                    step=0.1,
+                    label="Repeat Penalty",
+                    info="Penalize repetition"
+                )
+                
+                # Information section
+                gr.HTML("""
+                    <h3>ℹ️ About</h3>
+                    <p><strong>Format:</strong> GGUF (optimized)</p>
+                    <p><strong>Backend:</strong> llama-cpp-python</p>
+                    <p><strong>Features:</strong> CPU/GPU support, streaming</p>
+                    <p><strong>Memory:</strong> Optimized usage</p>
+                    <p><strong>Auto-Optimization:</strong> CPU threads & GPU layers detected automatically</p>
+                    <p><strong>Sources:</strong> Hugging Face Hub or Local Files</p>
+                """)
+                
+                if not LLAMA_CPP_AVAILABLE:
+                    gr.HTML("""
+                        <div style="background-color: #ffebee; padding: 10px; border-radius: 5px; margin-top: 10px;">
+                            <p style="color: #c62828; margin: 0;"><strong>⚠️ Missing Dependency</strong></p>
+                            <p style="color: #c62828; margin: 0; font-size: 0.9em;">
+                                Install llama-cpp-python:<br>
+                                <code>pip install llama-cpp-python</code>
+                            </p>
+                        </div>
+                    """)
+        
+        # Event handlers
+        def toggle_source_visibility(source_type):
+            if source_type == "Hugging Face":
+                return gr.update(visible=True), gr.update(visible=False)
+            else:
+                return gr.update(visible=False), gr.update(visible=True)
+        
+        source_type.change(
+            toggle_source_visibility,
+            inputs=source_type,
+            outputs=[hf_group, local_group]
+        )
+        
+        load_btn.click(
+            load_model_interface,
+            inputs=[source_type, gguf_dropdown, repo_id, filename, context_size],
+            outputs=model_status
+        )
+        
+        submit_btn.click(
+            chat_interface,
+            inputs=[msg, chatbot, max_tokens, temperature, top_p, repeat_penalty],
+            outputs=[chatbot, msg]
+        )
+        
+        msg.submit(
+            chat_interface,
+            inputs=[msg, chatbot, max_tokens, temperature, top_p, repeat_penalty],
+            outputs=[chatbot, msg]
+        )
+        
+        clear_btn.click(
+            clear_chat,
+            outputs=[chatbot, msg]
+        )
+        
+    return demo
+
+if __name__ == "__main__":
+    # Create and launch the interface
+    demo = create_interface()
+    
+    # Launch with appropriate settings for Hugging Face Spaces
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        debug=False,
+        show_error=True,
+        quiet=False
+    )