AMKCode commited on
Commit
8da6ea8
·
1 Parent(s): c869e4f

changed model card

Browse files
Files changed (1) hide show
  1. app.py +43 -5
app.py CHANGED
@@ -182,18 +182,56 @@ def button_click(hf_model_id, conv_template, quantization, oauth_token: gr.OAuth
182
 
183
  card.text = dedent(
184
  f"""
185
- # {created_repo_id}
186
- This model was compiled using MLC-LLM with {quantization} quantization from [{hf_model_id}]({HF_PATH}{hf_model_id}).
 
187
  The conversion was done using the [MLC-Weight-Conversion](https://huggingface.co/spaces/mlc-ai/MLC-Weight-Conversion) space.
 
 
 
188
 
189
- To run this model, please first install [MLC-LLM](https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages).
 
190
 
191
- To chat with the model on your terminal:
 
 
192
  ```bash
193
  mlc_llm chat HF://{created_repo_id}
194
  ```
195
 
196
- For more information on how to use MLC-LLM, please visit the MLC-LLM [documentation](https://llm.mlc.ai/docs/index.html).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
197
  """
198
  )
199
  card.save("./dist/README.md")
 
182
 
183
  card.text = dedent(
184
  f"""
185
+ # {mlc_model_name}
186
+
187
+ This is the [{model_dir_name}]({HF_PATH}{hf_model_id}) model in MLC format `e4m3_e4m3_f16` (FP8 quantization).
188
  The conversion was done using the [MLC-Weight-Conversion](https://huggingface.co/spaces/mlc-ai/MLC-Weight-Conversion) space.
189
+ The model can be used for projects [MLC-LLM](https://github.com/mlc-ai/mlc-llm).
190
+
191
+ ## Example Usage
192
 
193
+ Here are some examples of using this model in MLC LLM.
194
+ Before running the examples, please install MLC LLM by following the [installation documentation](https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages).
195
 
196
+ ### Chat
197
+
198
+ In command line, run
199
  ```bash
200
  mlc_llm chat HF://{created_repo_id}
201
  ```
202
 
203
+ ### REST Server
204
+
205
+ In command line, run
206
+ ```bash
207
+ mlc_llm serve HF://{created_repo_id}
208
+ ```
209
+
210
+ ### Python API
211
+
212
+ ```python
213
+ from mlc_llm import MLCEngine
214
+
215
+ # Create engine
216
+ model = "HF://{created_repo_id}"
217
+ engine = MLCEngine(model)
218
+
219
+ # Run chat completion in OpenAI API.
220
+ for response in engine.chat.completions.create(
221
+ messages=[{"role": "user", "content": "What is the meaning of life?"}],
222
+ model=model,
223
+ stream=True,
224
+ ):
225
+ for choice in response.choices:
226
+ print(choice.delta.content, end="", flush=True)
227
+ print("\n")
228
+
229
+ engine.terminate()
230
+ ```
231
+
232
+ ## Documentation
233
+
234
+ For more information on MLC LLM project, please visit our [documentation](https://llm.mlc.ai/docs/) and [GitHub repo](http://github.com/mlc-ai/mlc-llm).
235
  """
236
  )
237
  card.save("./dist/README.md")