File size: 1,404 Bytes
ef9c5ef c06d40f ef9c5ef |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
from transformers import (
AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline
)
import torch, os
MODEL_ID = "Qwen/Qwen3-32B" # 换成自己的模型
def get_model():
# ① 先试 bfloat16，A100/H100 都原生支持
return AutoModelForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
device_map="auto", # TGI 同款逻辑，自动分片
low_cpu_mem_usage=True, # 先在 CPU 建图，再流式拷到 GPU
trust_remote_code=True
)
# ---- 如果 bfloat16 仍 OOM，可改成 4-bit 量化 ----
# bnb_cfg = BitsAndBytesConfig(
# load_in_4bit=True,
# bnb_4bit_quant_type="nf4",
# bnb_4bit_use_double_quant=True,
# )
# def get_model():
# return AutoModelForCausalLM.from_pretrained(
# MODEL_ID,
# device_map="auto",
# quantization_config=bnb_cfg,
# trust_remote_code=True
# )
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
model = get_model()
generator = pipeline(
"text-generation",
model=model,
tokenizer=tokenizer,
device_map="auto",
torch_dtype=getattr(model, "dtype", torch.bfloat16),
)
def __init__(self, *args, **kwargs):
pass
def __call__(self, data):
prompt = data.get("inputs") if isinstance(data, dict) else data
outputs = generator(prompt, max_new_tokens=256)
return outputs
|