# import gradio as gr # from huggingface_hub import InferenceClient # """ # For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference # """ # client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") # ## None type # def respond( # message: str, # history: list[tuple[str, str]], # This will not be used # system_message: str, # max_tokens: int, # temperature: float, # top_p: float, # ): # messages = [{"role": "system", "content": system_message}] # # Append only the latest user message # messages.append({"role": "user", "content": message}) # response = "" # try: # # Generate response from the model # for message in client.chat_completion( # messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # if message.choices[0].delta.content is not None: # token = message.choices[0].delta.content # response += token # yield response # except Exception as e: # yield f"An error occurred: {e}" # ], # ) # if __name__ == "__main__": # demo.launch() ##Running smothly CHATBOT # import gradio as gr # from huggingface_hub import InferenceClient # """ # For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference # """ # client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") # def respond( # message: str, # history: list[tuple[str, str]], # This will not be used # system_message: str, # max_tokens: int, # temperature: float, # top_p: float, # ): # # Build the messages list # messages = [{"role": "system", "content": system_message}] # messages.append({"role": "user", "content": message}) # response = "" # try: # # Generate response from the model # for msg in client.chat_completion( # messages=messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # if msg.choices[0].delta.content is not None: # token = msg.choices[0].delta.content # response += token # yield response # except Exception as e: # yield f"An error occurred: {e}" # """ # For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface # """ # demo = gr.ChatInterface( # respond, # additional_inputs=[ # gr.Textbox(value="You are a friendly Chatbot.", label="System message"), # gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), # gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), # gr.Slider( # minimum=0.1, # maximum=1.0, # value=0.95, # step=0.05, # label="Top-p (nucleus sampling)", # ), # ], # ) # if __name__ == "__main__": # demo.launch() ### 26 aug Use a pipeline as a high-level Logic # import spaces # import os # import subprocess # from llama_cpp import Llama # from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType # from llama_cpp_agent.providers import LlamaCppPythonProvider # from llama_cpp_agent.chat_history import BasicChatHistory # from llama_cpp_agent.chat_history.messages import Roles # import gradio as gr # from huggingface_hub import hf_hub_download # huggingface_token = os.getenv("HF_TOKEN") # # Download the Meta-Llama-3.1-8B-Instruct model # hf_hub_download( # repo_id="bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", # filename="Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf", # local_dir="./models", # token=huggingface_token # ) # llm = None # llm_model = None # @spaces.GPU(duration=120) # def respond( # message, # history: list[tuple[str, str]], # model, # system_message, # max_tokens, # temperature, # top_p, # top_k, # repeat_penalty, # ): # chat_template = MessagesFormatterType.GEMMA_2 # global llm # global llm_model # # Load model only if it's not already loaded or if a new model is selected # if llm is None or llm_model != model: # try: # llm = Llama( # model_path=f"models/{model}", # flash_attn=True, # n_gpu_layers=81, # Adjust based on available GPU resources # n_batch=1024, # n_ctx=8192, # ) # llm_model = model # except Exception as e: # return f"Error loading model: {str(e)}" # provider = LlamaCppPythonProvider(llm) # agent = LlamaCppAgent( # provider, # system_prompt=f"{system_message}", # predefined_messages_formatter_type=chat_template, # debug_output=True # ) # settings = provider.get_provider_default_settings() # settings.temperature = temperature # settings.top_k = top_k # settings.top_p = top_p # settings.max_tokens = max_tokens # settings.repeat_penalty = repeat_penalty # settings.stream = True # messages = BasicChatHistory() # # Add user and assistant messages to the history # for msn in history: # user = {'role': Roles.user, 'content': msn[0]} # assistant = {'role': Roles.assistant, 'content': msn[1]} # messages.add_message(user) # messages.add_message(assistant) # # Stream the response # try: # stream = agent.get_chat_response( # message, # llm_sampling_settings=settings, # chat_history=messages, # returns_streaming_generator=True, # print_output=False # ) # outputs = "" # for output in stream: # outputs += output # yield outputs # except Exception as e: # yield f"Error during response generation: {str(e)}" # description = """

Using the Meta-Llama-3.1-8B-Instruct Model

""" # demo = gr.ChatInterface( # respond, # additional_inputs=[ # gr.Dropdown([ # 'Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf' # ], # value="Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf", # label="Model" # ), # gr.Textbox(value="You are a helpful assistant.", label="System message"), # gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"), # gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), # gr.Slider( # minimum=0.1, # maximum=1.0, # value=0.95, # step=0.05, # label="Top-p", # ), # gr.Slider( # minimum=0, # maximum=100, # value=40, # step=1, # label="Top-k", # ), # gr.Slider( # minimum=0.0, # maximum=2.0, # value=1.1, # step=0.1, # label="Repetition penalty", # ), # ], # retry_btn="Retry", # undo_btn="Undo", # clear_btn="Clear", # submit_btn="Send", # title="Chat with Meta-Llama-3.1-8B-Instruct using llama.cpp", # description=description, # chatbot=gr.Chatbot( # scale=1, # likeable=False, # show_copy_button=True # ) # ) # if __name__ == "__main__": # demo.launch() ####03 3.1 8b # import os # import time # import spaces # import torch # from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig # import gradio as gr # from threading import Thread # MODEL_LIST = ["meta-llama/Meta-Llama-3.1-8B-Instruct"] # HF_TOKEN = os.environ.get("HF_API_TOKEN",None) # print(HF_TOKEN,"######$$$$$$$$$$$$$$$") # MODEL = os.environ.get("MODEL_ID","meta-llama/Meta-Llama-3.1-8B-Instruct") # TITLE = "

Meta-Llama3.1-8B

" # PLACEHOLDER = """ #
#

Hi! How can I help you today?

#
# """ # CSS = """ # .duplicate-button { # margin: auto !important; # color: white !important; # background: black !important; # border-radius: 100vh !important; # } # h3 { # text-align: center; # } # """ # device = "cuda" # for GPU usage or "cpu" for CPU usage # quantization_config = BitsAndBytesConfig( # load_in_4bit=True, # bnb_4bit_compute_dtype=torch.bfloat16, # bnb_4bit_use_double_quant=True, # bnb_4bit_quant_type= "nf4") # tokenizer = AutoTokenizer.from_pretrained(MODEL) # model = AutoModelForCausalLM.from_pretrained( # MODEL, # torch_dtype=torch.bfloat16, # device_map="auto", # quantization_config=quantization_config) # @spaces.GPU() # def stream_chat( # message: str, # history: list, # system_prompt: str, # temperature: float = 0.8, # max_new_tokens: int = 1024, # top_p: float = 1.0, # top_k: int = 20, # penalty: float = 1.2, # ): # print(f'message: {message}') # print(f'history: {history}') # conversation = [ # {"role": "system", "content": system_prompt} # ] # for prompt, answer in history: # conversation.extend([ # {"role": "user", "content": prompt}, # {"role": "assistant", "content": answer}, # ]) # conversation.append({"role": "user", "content": message}) # input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device) # streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) # generate_kwargs = dict( # input_ids=input_ids, # max_new_tokens = max_new_tokens, # do_sample = False if temperature == 0 else True, # top_p = top_p, # top_k = top_k, # temperature = temperature, # repetition_penalty=penalty, # eos_token_id=[128001,128008,128009], # streamer=streamer, # ) # with torch.no_grad(): # thread = Thread(target=model.generate, kwargs=generate_kwargs) # thread.start() # buffer = "" # for new_text in streamer: # buffer += new_text # yield buffer # chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER) # with gr.Blocks(css=CSS, theme="soft") as demo: # gr.HTML(TITLE) # gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button") # gr.ChatInterface( # fn=stream_chat, # chatbot=chatbot, # fill_height=True, # additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), # additional_inputs=[ # gr.Textbox( # value="You are a helpful assistant", # label="System Prompt", # render=False, # ), # gr.Slider( # minimum=0, # maximum=1, # step=0.1, # value=0.8, # label="Temperature", # render=False, # ), # gr.Slider( # minimum=128, # maximum=8192, # step=1, # value=1024, # label="Max new tokens", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=1.0, # step=0.1, # value=1.0, # label="top_p", # render=False, # ), # gr.Slider( # minimum=1, # maximum=20, # step=1, # value=20, # label="top_k", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=2.0, # step=0.1, # value=1.2, # label="Repetition penalty", # render=False, # ), # ], # examples=[ # ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], # ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], # ["Tell me a random fun fact about the Roman Empire."], # ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], # ], # cache_examples=False, # ) # if __name__ == "__main__": # demo.launch() ###########new clientkey 04 ruunng chlrhah # import os # import time # import spaces # import torch # from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer # import gradio as gr # from threading import Thread # MODEL = "THUDM/LongWriter-llama3.1-8b" # TITLE = "

AreaX LLC-llama3.1-8b

" # PLACEHOLDER = """ #
#

Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?

#
# """ # CSS = """ # .duplicate-button { # margin: auto !important; # color: white !important; # background: black !important; # border-radius: 100vh !important; # } # h3 { # text-align: center; # } # """ # device = "cuda" if torch.cuda.is_available() else "cpu" # tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True) # model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto") # model = model.eval() # @spaces.GPU() # def stream_chat( # message: str, # history: list, # system_prompt: str, # temperature: float = 0.5, # max_new_tokens: int = 32768, # top_p: float = 1.0, # top_k: int = 50, # ): # print(f'message: {message}') # print(f'history: {history}') # full_prompt = f"<>\n{system_prompt}\n<>\n\n" # for prompt, answer in history: # full_prompt += f"[INST]{prompt}[/INST]{answer}" # full_prompt += f"[INST]{message}[/INST]" # inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device) # context_length = inputs.input_ids.shape[-1] # streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) # generate_kwargs = dict( # inputs=inputs.input_ids, # max_new_tokens=max_new_tokens, # do_sample=True, # top_p=top_p, # top_k=top_k, # temperature=temperature, # num_beams=1, # streamer=streamer, # ) # thread = Thread(target=model.generate, kwargs=generate_kwargs) # thread.start() # buffer = "" # for new_text in streamer: # buffer += new_text # yield buffer # chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER) # with gr.Blocks(css=CSS, theme="soft") as demo: # gr.HTML(TITLE) # gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button") # gr.ChatInterface( # fn=stream_chat, # chatbot=chatbot, # fill_height=True, # additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), # additional_inputs=[ # gr.Textbox( # value="You are a helpful assistant capable of generating long-form content.", # label="System Prompt", # render=False, # ), # gr.Slider( # minimum=0, # maximum=1, # step=0.1, # value=0.5, # label="Temperature", # render=False, # ), # gr.Slider( # minimum=1024, # maximum=32768, # step=1024, # value=32768, # label="Max new tokens", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=1.0, # step=0.1, # value=1.0, # label="Top p", # render=False, # ), # gr.Slider( # minimum=1, # maximum=100, # step=1, # value=50, # label="Top k", # render=False, # ), # ], # # examples=[ # # ["Write a 5000-word comprehensive guide on machine learning for beginners."], # # ["Create a detailed 3000-word business plan for a sustainable energy startup."], # # ["Compose a 2000-word short story set in a futuristic underwater city."], # # ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."], # # ], # # cache_examples=False, # ) # if __name__ == "__main__": # demo.launch() ###04 from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer from PIL import Image import requests import torch from threading import Thread import gradio as gr from gradio import FileData import time import spaces ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct" model = MllamaForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16).to("cuda") processor = AutoProcessor.from_pretrained(ckpt) @spaces.GPU def bot_streaming(message, history, max_new_tokens=250): txt = message["text"] ext_buffer = f"{txt}" messages= [] images = [] for i, msg in enumerate(history): if isinstance(msg[0], tuple): messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]}) messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]}) images.append(Image.open(msg[0][0]).convert("RGB")) elif isinstance(history[i-1], tuple) and isinstance(msg[0], str): # messages are already handled pass elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]}) messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]}) # add current message if len(message["files"]) == 1: if isinstance(message["files"][0], str): # examples image = Image.open(message["files"][0]).convert("RGB") else: # regular input image = Image.open(message["files"][0]["path"]).convert("RGB") images.append(image) messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]}) else: messages.append({"role": "user", "content": [{"type": "text", "text": txt}]}) texts = processor.apply_chat_template(messages, add_generation_prompt=True) if images == []: inputs = processor(text=texts, return_tensors="pt").to("cuda") else: inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda") streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True) generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens) generated_text = "" thread = Thread(target=model.generate, kwargs=generation_kwargs) thread.start() buffer = "" for new_text in streamer: buffer += new_text generated_text_without_prompt = buffer time.sleep(0.01) yield buffer demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama", examples=[ [{"text": "Which era does this piece belong to? Give details about the era.", "files":["./examples/rococo.jpg"]}, 200], [{"text": "Where do the droughts happen according to this diagram?", "files":["./examples/weather_events.png"]}, 250], [{"text": "What happens when you take out white cat from this chain?", "files":["./examples/ai2d_test.jpg"]}, 250], [{"text": "How long does it take from invoice date to due date? Be short and concise.", "files":["./examples/invoice.png"]}, 250], [{"text": "Where to find this monument? Can you give me other recommendations around the area?", "files":["./examples/wat_arun.jpg"]}, 250], ], textbox=gr.MultimodalTextbox(), additional_inputs = [gr.Slider( minimum=10, maximum=500, value=250, step=10, label="Maximum number of new tokens to generate", ) ], cache_examples=False, description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ", stop_btn="Stop Generation", fill_height=True, multimodal=True) demo.launch(debug=True)