Update app.py

app.py

@@ -16,6 +16,7 @@ def format_prompt(message, history):
 def generate(prompt, history, system_prompt, temperature=0.9, max_new_tokens=9048, top_p=0.95, repetition_penalty=1.0):
     temperature = max(float(temperature), 1e-2)
     top_p = float(top_p)
+
     generate_kwargs = dict(
         temperature=temperature,
         max_new_tokens=max_new_tokens,
@@ -31,11 +32,15 @@ def generate(prompt, history, system_prompt, temperature=0.9, max_new_tokens=904
     system_prompt = f"server log: ~This message was sent at {formatted_time}. The actual year is 2024.~"
 
     formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
-    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
-    output = ""
-    for response in stream:
-        output += response.token.text
-    yield output
+
+    # Use a loading indicator while the model is generating the response
+    with gr.Blocks().loading_indicator():
+        stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield output
+
     return output
 
 additional_inputs = [
@@ -54,9 +59,9 @@ chat_interface = gr.ChatInterface(
     chatbot=gr.Chatbot(show_label=True, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
     additional_inputs=additional_inputs,
     title="ConvoLite",
-    concurrency_limit=20,
+    concurrency_limit=50, # Increase the concurrency limit
 )
 
 with gr.Blocks(css=f"file={css_path}", theme=gr.themes.Soft()) as demo:
     demo.add(chat_interface)
-    demo.launch(show_api=False)
+    demo.launch(show_api=False)