|
import spaces |
|
import torch |
|
|
|
import gradio as gr |
|
from huggingface_hub import snapshot_download |
|
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig |
|
|
|
|
|
model = None |
|
model_id = "nazimali/Mistral-Nemo-Kurdish-Instruct" |
|
|
|
infer_prompt = """Li jêr rêwerzek heye ku peywirek rave dike, bi têketinek ku çarçoveyek din peyda dike ve tê hev kirin. Bersivek ku daxwazê bi guncan temam dike binivîsin. |
|
### Telîmat: |
|
{} |
|
### Têketin: |
|
{} |
|
### Bersiv: |
|
""" |
|
|
|
snapshot_download("nazimali/Mistral-Nemo-Kurdish") |
|
snapshot_download(repo_id=model_id) |
|
|
|
|
|
@spaces.GPU |
|
def respond( |
|
message, |
|
history: list[tuple[str, str]], |
|
): |
|
global model |
|
|
|
if model is None: |
|
bnb_config = BitsAndBytesConfig( |
|
load_in_4bit=True, |
|
bnb_4bit_use_double_quant=True, |
|
bnb_4bit_quant_type="nf4", |
|
bnb_4bit_compute_dtype=torch.bfloat16, |
|
) |
|
|
|
model = AutoModelForCausalLM.from_pretrained( |
|
model_id, |
|
quantization_config=bnb_config, |
|
device_map="auto", |
|
) |
|
tokenizer = AutoTokenizer.from_pretrained(model_id) |
|
|
|
model.eval() |
|
|
|
prompt = infer_prompt.format("tu arîkarek alîkar î", message) |
|
|
|
input_ids = tokenizer( |
|
prompt, |
|
return_tensors="pt", |
|
add_special_tokens=False, |
|
return_token_type_ids=False, |
|
).to("cuda") |
|
|
|
with torch.inference_mode(): |
|
generated_ids = model.generate( |
|
**input_ids, |
|
max_new_tokens=120, |
|
do_sample=True, |
|
temperature=0.7, |
|
top_p=0.7, |
|
num_return_sequences=1, |
|
pad_token_id=tokenizer.pad_token_id, |
|
eos_token_id=tokenizer.eos_token_id, |
|
) |
|
|
|
decoded_output = tokenizer.batch_decode(generated_ids)[0] |
|
|
|
return decoded_output.replace(prompt, "").replace("</s>", "") |
|
|
|
|
|
demo = gr.ChatInterface(respond, examples=["سڵاو ئەلیکوم، چۆنیت؟", "Selam alikum, tu çawa yî?"], title="Mistral Nemo Kurdish Instruct") |
|
|
|
|
|
if __name__ == "__main__": |
|
demo.launch() |
