import json from openai import OpenAI from typing import Any, Sequence, Generator from llama_index.core.base.llms.types import ( ChatMessage, ChatResponse, ) from llama_index.core.prompts import BasePromptTemplate from prompts.default_prompts import LAN_EXTRACT_PROMPT, FINAL_RESPONSE_PROMPT TokenGen = Generator[str, None, None] class MyVllm: def __init__(self, model: str, api_key: str, base_url: str, *args, **kwargs): self.client = OpenAI( base_url=base_url, api_key=api_key, ) self.model = model def chat(self, fmt_messages: Sequence[ChatMessage]) -> ChatResponse: completion = self.client.chat.completions.create( model=self.model, messages=[ {"role": fmt_message.role.value, "content": fmt_message.content} for fmt_message in fmt_messages ], temperature=0.8, max_tokens=5000, ) return ChatResponse( message=ChatMessage( content=completion.choices[0].message.content, role=completion.choices[0].message.role, ) ) def stream( self, prompt: BasePromptTemplate, **prompt_args: Any, ) -> TokenGen: fmt_messages = prompt.format_messages(**prompt_args) streaming = self.client.chat.completions.create( model=self.model, messages=[ {"role": fmt_message.role.value, "content": fmt_message.content} for fmt_message in fmt_messages ], stream=True, ) def gen(): for chunk in streaming: if chunk.choices[0].delta.content is not None: yield chunk.choices[0].delta.content return gen() if __name__ == "__main__": llm = MyVllm( model="huihui-ai/Qwen2.5-7B-Instruct-abliterated-v2", api_key="token-abc123", base_url="http://localhost:8000/v1", ) fmt_messages = LAN_EXTRACT_PROMPT.format_messages(user_input="give me some porn") response = llm.chat(fmt_messages=fmt_messages) print(response.message.content) video_result = { "videoResults": [ { "position": 1, "title": "Sexy Step Mom Fucked by Her Son", "link": "https://www.xvideos.com/video.upmulcbca89/sexy_step_mom_fucked_by_her_son", "displayed_link": "www.xvideos.com \u203a video.upmulcbca89 \u203a sexy_step_mo...", "thumbnail": "https://serpapi.com/searches/672c288f8b7ba7737b5abe6c/images/52fbc663a8c4062b682643f5f9d5f60301963876975fba3a4525a3e6064adb82.jpeg", "date": "Dec 13, 2022", "snippet": "Sexy Step Mom Fucked by Her Son 8 min 1440p. Tommy Wood \u00b7 dildo \u00b7 hardcore \u00b7 latina \u00b7 milf \u00b7 doggystyle \u00b7 homemade \u00b7 mature \u00b7 squirt \u00b7 dick ...", "duration": "8:12", "rich_snippet": { "top": { "detected_extensions": {"date": "Dec 13, 2022"}, "extensions": ["XVideos", "Dec 13, 2022"], } }, }, { "position": 2, "title": "Worlds Hottest Mom Fucks Son-REAL MOM SON 13:59", "link": "https://www.youjizz.com/videos/worlds-hottest-mom-fucks-son-real-mom-son-50905011.html", "displayed_link": "www.youjizz.com \u203a videos \u203a worlds-hottest-mom-fucks-s...", "thumbnail": "https://serpapi.com/searches/672c288f8b7ba7737b5abe6c/images/52fbc663a8c4062b041d2aa5f9f8f4dbaa236bb07770b0122b6648a85cb173a0.jpeg", "date": "Sep 27, 2018", "snippet": "Worlds Hottest Mom Fucks Son-REAL MOM SON 13:59. Tags: hot. MORE LESS. 362. 360. 288. 240. Play; Mute; Fullscreen. logo. Advertisement. Continue to video.", "rich_snippet": { "top": { "detected_extensions": {"date": "Sep 27, 2018"}, "extensions": ["Youjizz.com", "Sep 27, 2018"], } }, }, { "position": 3, "title": "Horny Stepson fucking his sexy stepmom once again before ...", "link": "https://www.xvideos.com/video.kfdilvb0e36/horny_stepson_fucking_his_sexy_stepmom_once_again_before_leaving_-_stepmom_step_mom-son_mom-fuck_mom-pov_mom-porn_mom-fucks-son_mom-porn_mother-son_mother-son-sex", "displayed_link": "www.xvideos.com \u203a horny_stepson_fucking_his_sexy_st...", "thumbnail": "https://serpapi.com/searches/672c288f8b7ba7737b5abe6c/images/52fbc663a8c4062b1be78d15de8d17e334242535e920e8ac2a57946ca7997157.jpeg", "date": "Dec 6, 2021", "snippet": "Horny Stepson fucking his sexy stepmom once again before leaving - stepmom step mom-son mom-fuck mom-pov mom-porn mom-fucks-son mom-porn mother-son mother-son- ...", "duration": "6:07", "rich_snippet": { "top": { "detected_extensions": {"date": "Dec 6, 2021"}, "extensions": [ "Free Porn Videos", "XVIDEOS.COM", "Dec 6, 2021", ], } }, }, { "position": 4, "title": "Sexy stepmom Helena cums to step sons rescue - stepmom ...", "link": "https://www.xvideos.com/video.kfdioeva2f9/sexy_stepmom_helena_cums_to_step_sons_rescue_-_stepmom_mom-son_mom-fuck_mom-pov_mom-porn_mom-fucks-son_mom-porn_mother-son_mother-son-sex", "displayed_link": "www.xvideos.com \u203a sexy_stepmom_helena_cums_to_ste...", "thumbnail": "https://serpapi.com/searches/672c288f8b7ba7737b5abe6c/images/52fbc663a8c4062b27d1fd6f6d999a416a504fcfa5d51f5faf1b4518d651db98.jpeg", "date": "Dec 6, 2021", "snippet": "Sexy stepmom Helena cums to step sons rescue - stepmom mom-son mom-fuck mom-pov mom-porn mom-fucks-son mom-porn mother-son mother-son-sex 7 min 720p", "duration": "7:14", "rich_snippet": { "top": { "detected_extensions": {"date": "Dec 6, 2021"}, "extensions": ["XVideos", "Dec 6, 2021"], } }, }, { "position": 5, "title": "Hot mom fucking the hell out of her step son.", "link": "https://www.xvideos.com/video.iatbvik37e0/hot_mom_fucking_the_hell_out_of_her_step_son.", "displayed_link": "www.xvideos.com \u203a hot_mom_fucking_the_hell_out_of...", "thumbnail": "https://serpapi.com/searches/672c288f8b7ba7737b5abe6c/images/52fbc663a8c4062b7260111894adc18d89d114b048eb96144e901b7de7c405cc.jpeg", "date": "Jan 17, 2019", "snippet": "Hot mom fucking the hell out of her step son. 7 min 720p. Free Black Porn \u00b7 Julianna Vega \u00b7 hardcore \u00b7 milf \u00b7 doggystyle \u00b7 mature \u00b7 mom ...", "duration": "6:59", "rich_snippet": { "top": { "detected_extensions": {"date": "Jan 17, 2019"}, "extensions": ["XVideos", "Jan 17, 2019"], } }, }, ] } response = llm.stream(FINAL_RESPONSE_PROMPT, search_result=json.dumps(video_result), search_keyword="Sexy Mom Fuck Son", language='english') for token in response: print(token + '\n\n')