Quantization
Browse files- handler.py +2 -1
handler.py
CHANGED
|
@@ -4,7 +4,8 @@ from typing import Dict, List, Any
|
|
| 4 |
|
| 5 |
class EndpointHandler():
|
| 6 |
def __init__(self, path=""):
|
| 7 |
-
self.
|
|
|
|
| 8 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
| 9 |
self.pipeline = pipeline("text-generation", model=self.model, tokenizer=self.tokenizer)
|
| 10 |
|
|
|
|
| 4 |
|
| 5 |
class EndpointHandler():
|
| 6 |
def __init__(self, path=""):
|
| 7 |
+
self.quant = quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16)
|
| 8 |
+
self.model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", quantization_config=self.quant, trust_remote_code=True)
|
| 9 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
| 10 |
self.pipeline = pipeline("text-generation", model=self.model, tokenizer=self.tokenizer)
|
| 11 |
|