changing to my model with transformers

app.py

@@ -1,11 +1,9 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-#client = InferenceClient("michailroussos/model-mistral_CP1250")
+client = AutoModelForCausalLM.from_pretrained("michailroussos/model-mistral_CP1250", torch_dtype=torch.float16)
+tokenizer = AutoTokenizer.from_pretrained("michailroussos/model-mistral_CP1250")
 
 def respond(
     message,
@@ -25,18 +23,28 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
+    # Convert messages to prompt
+    prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
+    # Generate response
+    inputs = tokenizer(prompt, return_tensors="pt").to(client.device)
+    
+    response = ""
+    for _ in range(max_tokens):
+        with torch.no_grad():
+            outputs = client.generate(
+                inputs.input_ids, 
+                max_new_tokens=1,
+                temperature=temperature,
+                top_p=top_p,
+                do_sample=True,
+            )
+        
+        new_token = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+        response += new_token
+        
+        inputs = tokenizer(inputs.input_ids.tolist()[0] + outputs[0][inputs.input_ids.shape[1]:].tolist(), return_tensors="pt")
+        
         yield response
 
 

requirements.txt

@@ -1 +1,3 @@
-huggingface_hub==0.25.2
+huggingface_hub==0.25.2
+transformers
+torch