nito78/llm-jp-3-3.7b-it_lora_all

LLM-JP-3.3.7B LoRA Model

This is the LLM-JP-3.3.7B model fine-tuned with LoRA for instruction-based Japanese text generation tasks. The model has been fine-tuned on datasets ichikara-instruction and Ego/jpflan-raw.

How to Use

Below is an example of how to use the model for inference:

import torch
from unsloth import FastLanguageModel
from peft import PeftModel

HF_TOKEN = ""  # Add your Hugging Face token here

# Load the base model and tokenizer
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="llm-jp/llm-jp-3-3.7b",
    dtype=None,
    load_in_4bit=True,
    trust_remote_code=True,
)
model = PeftModel.from_pretrained(model, "nito78/llm-jp-3-3.7b-it_lora_all", token=HF_TOKEN)

# Switch to inference mode
FastLanguageModel.for_inference(model)

# Example usage
import json

# Load dataset
datasets = []
with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
    item = ""
    for line in f:
        line = line.strip()
        item += line
        if item.endswith("}"):
            datasets.append(json.loads(item))
            item = ""

from tqdm import tqdm

# Perform inference
results = []
for dt in tqdm(datasets):
    input = dt["input"]

    prompt = f"""### 指示\n{input}\n### 回答\n"""

    inputs = tokenizer([prompt], return_tensors="pt").to(model.device)

    outputs = model.generate(**inputs, max_new_tokens=512, use_cache=True, do_sample=False, repetition_penalty=1.2)
    prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split("\n### 回答")[-1]

    results.append({"task_id": dt["task_id"], "input": input, "output": prediction})

import os
import json

# Save results
output_dir = "./results"
os.makedirs(output_dir, exist_ok=True)

output_file = os.path.join(output_dir, "result.jsonl")
with open(output_file, "w", encoding="utf-8") as f:
    for result in results:
        json.dump(result, f, ensure_ascii=False)
        f.write("\n")