update

app.py

@@ -1,57 +1,90 @@
+
+
+import spaces
 import os
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 import gradio as gr
-import spaces
+
 
 huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
 if not huggingface_token:
     pass
-    raise ValueError("HUGGINGFACE_TOKEN environment variable is not set")
+    print("no HUGGINGFACE_TOKEN if you need set secret ")
+    #raise ValueError("HUGGINGFACE_TOKEN environment variable is not set")
 
 model_id = "google/gemma-2-9b-it"
-#model_id = "microsoft/Phi-3-mini-128k-instruct"
-# device_map style value auto not cuda
-device = "auto" #torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+device = "auto" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
 dtype = torch.bfloat16
 
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=huggingface_token)
 
-#print(model_id,device,dtype)
-@spaces.GPU
-def generate_text(prompt, system_message="You are a helpful assistant."):
+print(model_id,device,dtype)
+histories = []
+#model = None
+
+
+@spaces.GPU(duration=120)
+def generate_text(messages):
     model = AutoModelForCausalLM.from_pretrained(
-        model_id, torch_dtype=dtype,device_map=device, token=huggingface_token
+        model_id, token=huggingface_token ,torch_dtype=dtype,device_map=device
     )
-    text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer, torch_dtype=dtype, device_map=device)
-
-    messages = [
-        {"role": "system", "content": system_message},
-        {"role": "user", "content": prompt},
-    ]
 
+    text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer,torch_dtype=dtype,device_map=device) #pipeline has not to(device)
     result = text_generator(messages, max_new_tokens=256, do_sample=True, temperature=0.7)
 
     generated_output = result[0]["generated_text"]
     if isinstance(generated_output, list):
         for message in reversed(generated_output):
             if message.get("role") == "assistant":
-                return message.get("content", "No content found.")
+                content= message.get("content", "No content found.")
+                return content
+            
         return "No assistant response found."
     else:
         return "Unexpected output format."
 
+def call_generate_text(prompt, system_message="You are a helpful assistant."):
+    if prompt =="":
+        print("empty prompt return")
+        return ""
+    
+    global histories
+        
+    messages = [
+        {"role": "system", "content": system_message},
+    ]
+    
+    messages += histories 
+
+    user_message = {"role": "user", "content": prompt}
+
+    messages += [user_message]
+    
+    try:
+        text = generate_text(messages)
+        histories += [user_message,{"role": "assistant", "content": text}]
+       
+        return text
+    except RuntimeError  as e:
+        print(f"An unexpected error occurred: {e}")
+       
+    return ""
 
 iface = gr.Interface(
-    fn=generate_text,
+    fn=call_generate_text,
     inputs=[
         gr.Textbox(lines=3, label="Input Prompt"),
-        gr.Textbox(lines=2, label="System Message", value="You are a helpful assistant."),
+        gr.Textbox(lines=2, label="System Message", value="あなたは親切なアシスタントで常に日本語で返答します。"),
     ],
     outputs=gr.Textbox(label="Generated Text"),
-    title="google/gemma-2-9b-it Text Generation",
-    description="Enter a prompt and optional system message to generate text using the google/gemma-2-9b-it model.",
+    title=f"{model_id}",
+    description=f"{model_id}",
 )
+print("Initialized")
+
 
 if __name__ == "__main__":
+    print("Main")
     iface.launch()