Gabriel
/

Qwen2-VL-7B-Instruct-AWQ

Image-Text-to-Text

text-generation-inference

4-bit precision

Model card Files Files and versions Community

Gabriel commited on Sep 26, 2024

Commit

84aa6a7

·

verified ·

1 Parent(s): 1873b39

Update handler.py

Files changed (1) hide show

handler.py +1 -1

handler.py CHANGED Viewed

@@ -8,7 +8,7 @@ import requests
 class EndpointHandler():
     def __init__(self, path=""):
         self.processor = AutoProcessor.from_pretrained(path)
-        self.model = Qwen2VLForConditionalGeneration.from_pretrained(path)
     def __call__(self, data: Any) -> Dict[str, Any]:

 class EndpointHandler():
     def __init__(self, path=""):
         self.processor = AutoProcessor.from_pretrained(path)
+        self.model = Qwen2VLForConditionalGeneration.from_pretrained(path, device_map="cuda:0")
     def __call__(self, data: Any) -> Dict[str, Any]: