Create app.py

app.py

@@ -0,0 +1,117 @@
+import gradio as gr
+import torch
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    TextIteratorStreamer,
+)
+import os
+from threading import Thread
+import spaces
+import time
+import subprocess
+
+PLACEHOLDER = """
+<div style="padding: 40px; text-align: center; display: flex; flex-direction: column; align-items: center;">
+   <img src="https://i.imgur.com/dgSNbTl.jpg" style="width: 90%; max-width: 650px; height: auto; opacity: 0.8;  "> 
+   <h1 style="font-size: 28px; margin-top: 20px; margin-bottom: 2px; opacity: 0.55;">mii-llm / Maestrale</h1>
+   <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Chiedi pure...</p>
+</div>
+"""
+
+DESCRIPTION = """<div>
+<p>🇮🇹 Italian LLM <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.3-beta"><b>Maestrale Chat v0.3 beta</b></a>. Maestrale is a powerful language model for Italian, trained by mii-llm, based on Mistral 7B.</p>
+<p>🔎 For more details about the Maestrale model and how to use it with <code>transformers</code>, visit the <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.3-beta">model card</a>.</p>
+</div>"""
+
+tokenizer = AutoTokenizer.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha")
+model = AutoModelForCausalLM.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha", device_map="auto")
+
+terminators = [
+    tokenizer.eos_token_id,
+    tokenizer.convert_tokens_to_ids("<|im_end|>")
+]
+
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+    print(f"Using GPU: {torch.cuda.get_device_name(device)}")
+else:
+    device = torch.device("cpu")
+    print("Using CPU")
+
+
+model = model.to(device)
+
+
+@spaces.GPU(duration=60)
+def chat(message, history, system, temperature, do_sample, max_tokens):
+    chat = [{"role": "system", "content": system}] if system else []
+    chat.extend(
+        {"role": role, "content": content}
+        for user, assistant in history
+        for role, content in [("user", user), ("assistant", assistant)]
+    )
+    chat.append({"role": "user", "content": message})
+
+    messages = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+    model_inputs = tokenizer([messages], return_tensors="pt").to(device)
+
+    streamer = TextIteratorStreamer(
+        tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True
+    )
+
+    generate_kwargs = {
+        **model_inputs,
+        "streamer": streamer,
+        "max_new_tokens": max_tokens,
+        "do_sample": do_sample,
+        "temperature": temperature,
+        "eos_token_id": terminators,
+    }
+
+    thread = Thread(target=model.generate, kwargs=generate_kwargs)
+    thread.start()
+
+    partial_text = ""
+    for new_text in streamer:
+        partial_text += new_text
+        yield partial_text
+
+    yield partial_text
+
+
+chatbot = gr.Chatbot(height=550, placeholder=PLACEHOLDER, label='Conversazione')
+
+demo = gr.ChatInterface(
+    fn=chat,
+    chatbot=chatbot,
+    fill_height=True,
+    theme=gr.themes.Soft(),
+    additional_inputs_accordion=gr.Accordion(
+        label="⚙️ Parametri", open=False, render=False
+    ),
+    additional_inputs=[
+        gr.Textbox(
+            label="System",
+            value="sei un assistente utile.",
+        ),
+        gr.Slider(
+            minimum=0, maximum=1, step=0.1, value=0.7, label="Temperature", render=False
+        ),
+        gr.Checkbox(label="Sampling", value=True),
+        gr.Slider(
+            minimum=128,
+            maximum=4096,
+            step=1,
+            value=512,
+            label="Max new tokens",
+            render=False,
+        ),
+    ],
+    stop_btn="Stop Generation",
+    cache_examples=False,
+    title="Maestrale Chat v0.3 beta",
+    description=DESCRIPTION
+)
+
+demo.launch()