app.py

# import gradio as gr
# from huggingface_hub import InferenceClient

# """
# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
# """
# client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

# ## None type 
# def respond(
#     message: str,
#     history: list[tuple[str, str]],  # This will not be used
#     system_message: str,
#     max_tokens: int,
#     temperature: float,
#     top_p: float,
# ):
#     messages = [{"role": "system", "content": system_message}]
    
#     # Append only the latest user message





#     messages.append({"role": "user", "content": message})

#     response = ""

#     try:
#         # Generate response from the model
#         for message in client.chat_completion(
#             messages,
#             max_tokens=max_tokens,
#             stream=True,
#             temperature=temperature,
#             top_p=top_p,
#         ):
#             if message.choices[0].delta.content is not None:
#                 token = message.choices[0].delta.content
#                 response += token
#             yield response
#     except Exception as e:
#         yield f"An error occurred: {e}"
#     ],
# )


# if __name__ == "__main__":
#     demo.launch()


##Running smothly CHATBOT

# import gradio as gr
# from huggingface_hub import InferenceClient

# """
# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
# """
# client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

# def respond(
#     message: str,
#     history: list[tuple[str, str]],  # This will not be used
#     system_message: str,
#     max_tokens: int,
#     temperature: float,
#     top_p: float,
# ):
#     # Build the messages list
#     messages = [{"role": "system", "content": system_message}]
#     messages.append({"role": "user", "content": message})

#     response = ""

#     try:
#         # Generate response from the model
#         for msg in client.chat_completion(
#             messages=messages,
#             max_tokens=max_tokens,
#             stream=True,
#             temperature=temperature,
#             top_p=top_p,
#         ):
#             if msg.choices[0].delta.content is not None:
#                 token = msg.choices[0].delta.content
#                 response += token
#             yield response
#     except Exception as e:
#         yield f"An error occurred: {e}"

# """
# For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
# """
# demo = gr.ChatInterface(
#     respond,
#     additional_inputs=[
#         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
#         gr.Slider(
#             minimum=0.1,
#             maximum=1.0,
#             value=0.95,
#             step=0.05,
#             label="Top-p (nucleus sampling)",
#         ),
#     ],
# )

# if __name__ == "__main__":
#     demo.launch()


### 20aug

# import os
# import time
# import spaces
# import torch
# from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
# import gradio as gr
# from threading import Thread

# MODEL_LIST = ["meta-llama/Meta-Llama-3.1-8B-Instruct"]
# HF_TOKEN = os.environ.get("HF_API_TOKEN", None)
# MODEL = os.environ.get("MODEL_ID")

# TITLE = "<h1><center>Meta-Llama3.1-8B</center></h1>"

# PLACEHOLDER = """
# <center>
# <p>Hi! How can I help you today?</p>
# </center>
# """


# CSS = """
# .duplicate-button {
#     margin: auto !important;
#     color: white !important;
#     background: black !important;
#     border-radius: 100vh !important;
# }
# h3 {
#     text-align: center;
# }
# """

# device = "cuda" # for GPU usage or "cpu" for CPU usage

# quantization_config = BitsAndBytesConfig(
#     load_in_4bit=True,
#     bnb_4bit_compute_dtype=torch.bfloat16,
#     bnb_4bit_use_double_quant=True,
#     bnb_4bit_quant_type= "nf4")

# tokenizer = AutoTokenizer.from_pretrained(MODEL)
# model = AutoModelForCausalLM.from_pretrained(
#     MODEL,
#     torch_dtype=torch.bfloat16,
#     device_map="auto",
#     quantization_config=quantization_config)

# @spaces.GPU()
# def stream_chat(
#     message: str, 
#     history: list,
#     system_prompt: str,
#     temperature: float = 0.8, 
#     max_new_tokens: int = 1024, 
#     top_p: float = 1.0, 
#     top_k: int = 20, 
#     penalty: float = 1.2,
# ):
#     print(f'message: {message}')
#     print(f'history: {history}')

#     conversation = [
#         {"role": "system", "content": system_prompt}
#     ]
#     for prompt, answer in history:
#         conversation.extend([
#             {"role": "user", "content": prompt}, 
#             {"role": "assistant", "content": answer},
#         ])

#     conversation.append({"role": "user", "content": message})

#     input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
    
#     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
    
#     generate_kwargs = dict(
#         input_ids=input_ids, 
#         max_new_tokens = max_new_tokens,
#         do_sample = False if temperature == 0 else True,
#         top_p = top_p,
#         top_k = top_k,
#         temperature = temperature,
#         repetition_penalty=penalty,
#         eos_token_id=[128001,128008,128009],
#         streamer=streamer,
#     )

#     with torch.no_grad():
#         thread = Thread(target=model.generate, kwargs=generate_kwargs)
#         thread.start()
        
#     buffer = ""
#     for new_text in streamer:
#         buffer += new_text
#         yield buffer

            
# chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

# with gr.Blocks(css=CSS, theme="soft") as demo:
#     gr.HTML(TITLE)
#     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
#     gr.ChatInterface(
#         fn=stream_chat,
#         chatbot=chatbot,
#         fill_height=True,
#         additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
#         additional_inputs=[
#             gr.Textbox(
#                 value="You are a helpful assistant",
#                 label="System Prompt",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0,
#                 maximum=1,
#                 step=0.1,
#                 value=0.8,
#                 label="Temperature",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=128,
#                 maximum=8192,
#                 step=1,
#                 value=1024,
#                 label="Max new tokens",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0.0,
#                 maximum=1.0,
#                 step=0.1,
#                 value=1.0,
#                 label="top_p",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=1,
#                 maximum=20,
#                 step=1,
#                 value=20,
#                 label="top_k",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0.0,
#                 maximum=2.0,
#                 step=0.1,
#                 value=1.2,
#                 label="Repetition penalty",
#                 render=False,
#             ),
#         ],
#         examples=[
#             ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
#             ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
#             ["Tell me a random fun fact about the Roman Empire."],
#             ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
#         ],
#         cache_examples=False,
#     )


# if __name__ == "__main__":
#     demo.launch()

import os
import gradio as gr
from huggingface_hub import InferenceClient


# Your Hugging Face configuration
model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct"
# token = "hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

# Initialize Inference Client with model and token
inference_client = InferenceClient()

def chat_completion(message, history):
    # Pass user input through Hugging Face model
    response = inference_client.chat(
        model=model_name,
        messages=[{"role": "user", "content": message}],
        max_tokens=500,
        stream=False
    )
    
    # Extract content from the response
    response_text = response['choices'][0]['delta']['content']
    
    # Return response and updated history
    return response_text

# Create Gradio chat interface
chatbot = gr.ChatInterface(fn=chat_completion)
chatbot.launch()