Spaces:
Sleeping
Sleeping
File size: 1,711 Bytes
36942d4 852d26e cee13f4 852d26e 095073f 341bd22 852d26e 095073f 852d26e 341bd22 852d26e fc2aea6 852d26e 341bd22 852d26e 341bd22 852d26e 2918965 852d26e 341bd22 b56c1a4 852d26e 341bd22 852d26e 341bd22 852d26e 341bd22 852d26e 341bd22 37765a6 852d26e 341bd22 ea01ae0 852d26e 999df08 341bd22 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import os
import threading
import gradio as gr
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
TextIteratorStreamer,
)
MODEL_NAME = "MaxLSB/LeCarnet-8M"
hf_token = os.environ["HUGGINGFACEHUB_API_TOKEN"]
# Load tokenizer & model locally
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, token=hf_token)
model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, token=hf_token)
model.eval()
def respond(
prompt: str,
chat_history,
max_tokens: int,
temperature: float,
top_p: float,
):
inputs = tokenizer(prompt, return_tensors="pt")
streamer = TextIteratorStreamer(
tokenizer,
skip_prompt=True,
skip_special_tokens=True,
)
generate_kwargs = dict(
**inputs,
streamer=streamer,
max_new_tokens=max_tokens,
do_sample=True,
temperature=temperature,
top_p=top_p,
eos_token_id=tokenizer.eos_token_id,
)
thread = threading.Thread(target=model.generate, kwargs=generate_kwargs)
thread.start()
accumulated = ""
for new_text in streamer:
accumulated += new_text
yield accumulated
# Wire it up in Gradio
demo = gr.ChatInterface(
fn=respond,
additional_inputs=[
gr.Slider(1, 512, value=512, step=1, label="Max new tokens"),
gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature"),
gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top‑p"),
],
title="LeCarnet-8M: Short French Stories",
description="Type the beginning of a sentence and watch the model finish it.",
examples=[
["Il était une fois un petit garçon"],
],
)
if __name__ == "__main__":
demo.launch()
|