Update README.md

README.md

@@ -13,3 +13,41 @@ language:
 # huihui-ai/dots.llm1.inst
 
 This version only allows local loading of dots.llm1.inst using transformers, with only the local import issue modified and no other changes.
+
+## Usage
+Copy the four files to the model directory, and then you can use the following program.
+
+```
+import sys
+import os
+import torch
+from transformers import AutoTokenizer, AutoConfig, AutoModel, BitsAndBytesConfig
+
+AutoConfig.register("dots1", Dots1Config)
+AutoModel.register(Dots1Config, Dots1ForCausalLM)
+
+config = AutoConfig.from_pretrained(MODEL_ID)
+print(config)
+
+quant_config_4 = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    llm_int8_enable_fp32_cpu_offload=True,
+)
+
+model = Dots1ForCausalLM.from_pretrained(
+    MODEL_ID,
+    device_map="auto",
+    trust_remote_code=True,
+    quantization_config=quant_config_4,
+    torch_dtype=torch.bfloat16,
+    low_cpu_mem_usage=True,
+)
+
+print(model)
+print(model.config)
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
+
+```