try fp16

handler.py

@@ -18,9 +18,6 @@ logging.basicConfig(level=logging.INFO)
 hf_logging.set_verbosity_debug()
 
 
-# 2023/11/28 06:41:40 ~ loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730/generation_config.json
-
-
 def list_files(directory, depth, max_depth=5):
     # Lists all files and directories in the given directory
     for filename in os.listdir(directory):
@@ -58,7 +55,7 @@ class EndpointHandler:
         with init_empty_weights():
             self.model = AutoModelForCausalLM.from_pretrained(
                 "THUDM/cogvlm-chat-hf",
-                torch_dtype=torch.bfloat16,
+                torch_dtype=torch.float16,
                 low_cpu_mem_usage=True,
                 trust_remote_code=True,
             )
@@ -78,13 +75,9 @@ class EndpointHandler:
             no_split_module_classes=["CogVLMDecoderLayer"],
         )
 
-        # .cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730
         self.model = load_checkpoint_and_dispatch(
             self.model,
             "/root/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730",
-            # "~/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730",
-            # "/home/ec2-user/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/8abca878c4257412c4c38eeafaed3fe27a036730",  # typical, '~/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/balabala'
-            # "~/.cache/huggingface/modules/transformers_modules/THUDM/cogvlm-chat-hf/8abca878c4257412c4c38eeafaed3fe27a036730",  # typical, '~/.cache/huggingface/hub/models--THUDM--cogvlm-chat-hf/snapshots/balabala'
             device_map=device_map,
             no_split_module_classes=["CogVLMDecoderLayer"],
         )