update

app.py

@@ -8,6 +8,7 @@ import gradio as gr
 import spaces
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, pipeline
+from peft import PeftModel
 
 DESCRIPTION = "# chat-1"
 
@@ -22,10 +23,16 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "32768"))
 if torch.cuda.is_available():
     model_id = "vericava/llm-jp-3-1.8b-instruct-lora-vericava7-llama"
     base_model_id = "llm-jp/llm-jp-3-1.8b-instruct"
+    tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)
+    base_model = AutoModelForCausalLM.from_pretrained(
+        base_model_id,
+        trust_remote_code=True,
+    )
+    model = PeftModel.from_pretrained(base_model, model_id, trust_remote_code=True)
     my_pipeline=pipeline(
-        model=model_id,
-        tokenizer=base_model_id,
-        use_safetensors=True,
+        task="text-generation",
+        model=model,
+        tokenizer=tokenizer,
     )
     my_pipeline.tokenizer.chat_template = "{{bos_token}}{% for message in messages %}{% if message['role'] == 'user' %}{{ '\\n\\n### 前の投稿:\\n' + message['content'] + '' }}{% elif message['role'] == 'system' %}{{ '以下は、SNS上の投稿です。あなたはSNSの投稿生成botとして、次に続く投稿を考えなさい。説明はせず、投稿の内容のみを鉤括弧をつけずに答えよ。' }}{% elif message['role'] == 'assistant' %}{{ '\\n\\n### 次の投稿:\\n' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '\\n\\n### 次の投稿:\\n' }}{% endif %}{% endfor %}"
 

pyproject.toml

@@ -1,12 +1,13 @@
 [project]
 dependencies = [
-    "accelerate>=1.2.1",
-    "bitsandbytes>=0.45.0",
-    "gradio>=5.9.1",
-    "hf-transfer>=0.1.8",
-    "spaces>=0.31.1",
-    "torch==2.4.0",
-    "transformers>=4.48.3",
+  "accelerate>=1.2.1",
+  "bitsandbytes>=0.45.0",
+  "gradio>=5.9.1",
+  "hf-transfer>=0.1.8",
+  "spaces>=0.31.1",
+  "torch==2.4.0",
+  "transformers>=4.48.3",
+  "peft==0.14.0",
 ]
 description = ""
 name = "chat-1"

requirements.txt

@@ -1,7 +1,9 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml -o requirements.txt
 accelerate==1.2.1
-    # via calm2-7b-chat (pyproject.toml)
+    # via
+    #   chat-1 (pyproject.toml)
+    #   peft
 aiofiles==23.2.1
     # via gradio
 annotated-types==0.7.0
@@ -12,7 +14,7 @@ anyio==4.7.0
     #   httpx
     #   starlette
 bitsandbytes==0.45.0
-    # via calm2-7b-chat (pyproject.toml)
+    # via chat-1 (pyproject.toml)
 certifi==2024.12.14
     # via
     #   httpcore
@@ -41,7 +43,7 @@ fsspec==2024.12.0
     #   torch
 gradio==5.9.1
     # via
-    #   calm2-7b-chat (pyproject.toml)
+    #   chat-1 (pyproject.toml)
     #   spaces
 gradio-client==1.5.2
     # via gradio
@@ -50,7 +52,7 @@ h11==0.14.0
     #   httpcore
     #   uvicorn
 hf-transfer==0.1.8
-    # via calm2-7b-chat (pyproject.toml)
+    # via chat-1 (pyproject.toml)
 httpcore==1.0.7
     # via httpx
 httpx==0.28.1
@@ -64,6 +66,7 @@ huggingface-hub==0.27.0
     #   accelerate
     #   gradio
     #   gradio-client
+    #   peft
     #   tokenizers
     #   transformers
 idna==3.10
@@ -93,6 +96,7 @@ numpy==2.2.1
     #   bitsandbytes
     #   gradio
     #   pandas
+    #   peft
     #   transformers
 nvidia-cublas-cu12==12.1.3.1
     # via
@@ -133,15 +137,19 @@ packaging==24.2
     #   gradio
     #   gradio-client
     #   huggingface-hub
+    #   peft
     #   spaces
     #   transformers
 pandas==2.2.3
     # via gradio
+peft==0.14.0
+    # via chat-1 (pyproject.toml)
 pillow==11.1.0
     # via gradio
 psutil==5.9.8
     # via
     #   accelerate
+    #   peft
     #   spaces
 pydantic==2.10.4
     # via
@@ -165,6 +173,7 @@ pyyaml==6.0.2
     #   accelerate
     #   gradio
     #   huggingface-hub
+    #   peft
     #   transformers
 regex==2024.11.6
     # via transformers
@@ -182,6 +191,7 @@ safehttpx==0.1.6
 safetensors==0.4.5
     # via
     #   accelerate
+    #   peft
     #   transformers
 semantic-version==2.10.0
     # via gradio
@@ -192,7 +202,7 @@ six==1.17.0
 sniffio==1.3.1
     # via anyio
 spaces==0.31.1
-    # via calm2-7b-chat (pyproject.toml)
+    # via chat-1 (pyproject.toml)
 starlette==0.41.3
     # via
     #   fastapi
@@ -205,15 +215,19 @@ tomlkit==0.13.2
     # via gradio
 torch==2.4.0
     # via
-    #   calm2-7b-chat (pyproject.toml)
+    #   chat-1 (pyproject.toml)
     #   accelerate
     #   bitsandbytes
+    #   peft
 tqdm==4.67.1
     # via
     #   huggingface-hub
+    #   peft
     #   transformers
 transformers==4.50.1
-    # via calm2-7b-chat (pyproject.toml)
+    # via
+    #   chat-1 (pyproject.toml)
+    #   peft
 triton==3.0.0
     # via torch
 typer==0.15.1