i# This is a combination of 8 commits.

author qscqesze <[email protected]> 1748943522 +0000
committer qingjun <[email protected]> 1749004700 +0800

initial

.gitattributes

@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text

LICENSE-CODE

@@ -0,0 +1,15 @@
+
+MIT License
+
+Copyright 2025 MiniMax AI
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 
+documentation files (the “Software”), to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions 
+of the Software.
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 
+CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
+IN THE SOFTWARE.

LICENSE-MODEL

@@ -0,0 +1,43 @@
+
+MINIMAX MODEL LICENSE
+
+Model Release Date: 15 January 2025
+
+1. Definitions 
+"Agreement" means the terms and conditions for use, reproduction, distribution and modification of the Model Materials set forth herein.
+"Licensee" or "you" means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
+"Model" means the foundational large language models and software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code and other elements of the foregoing distributed by MiniMax at https://huggingface.co/MiniMaxAI and https://github.com/MiniMax-AI.
+"Model Materials" means, collectively, the Model and any source code, scripts, specifications, manuals and documentation accompanying the Model (and any portion thereof) made available under this Agreement.
+"MiniMax" or "we" means MiniMax AI. 
+
+2. License Rights and Redistribution
+   a. Grant of Rights. You are granted a non-exclusive, worldwide and royalty-free limited license under MiniMax’s intellectual property or other rights owned by MiniMax embodied in the Model Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Model Materials. 
+   b. Redistribution and Use.  
+      i. If you distribute or make available the Model Materials (or any derivative works thereof), or a product or service that uses any of them, including another AI model, you shall provide a copy of this Agreement with any such the Model Materials or derivative works and cause any modified files to carry prominent notices stating that you changed the files. You may add your own copyright statement to your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of your modifications, or for any such derivative works as a whole, provided your use, reproduction, and distribution of the work otherwise complies with the terms and conditions of this Agreement.
+      ii. You must retain in all copies of the Model Materials that you distribute the following attribution notice within a “Notice” text file distributed as a part of such copies: “MiniMax AI model is licensed under the MiniMax Model License, Copyright © MiniMax. All Rights Reserved.”
+      iii. Your use of the Model Materials must comply with applicable laws and regulations (including trade compliance laws and regulations) and adhere to the Prohibited Uses Policy for the Model Materials, which is hereby incorporated by reference into this Agreement.  
+
+3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE MODEL MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND MINIMAX DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE MODEL MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE MODEL MATERIALS AND ANY OUTPUT AND RESULTS.
+
+4. Limitation of Liability. IN NO EVENT WILL MINIMAX OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF MINIMAX OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+
+5. Intellectual Property.
+a. No trademark license is granted to use the trade names, trademarks, service marks, or product names of MiniMax, except as required to fulfill notice requirements under this Agreement.   
+b. Subject to MiniMax’s ownership of the Model Materials and derivatives made by or for MiniMax, with respect to any derivative works and modifications of the Model Materials that are made by you, as between you and MiniMax, you are and will be the owner of such derivative works and modifications.
+c. If you institute litigation or other proceedings against MiniMax or any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Model Materials or outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless MiniMax from and against any claim by any third party arising out of or related to your use or distribution of the Model Materials.
+
+6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Model Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. MiniMax may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the Model Materials. Sections 3, 4 and 7 shall survive the termination of this Agreement. 
+ 
+7. Governing Law and Jurisdiction. This Agreement will be governed and construed under the laws of Singapore without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. Any dispute arising out of or in connection with this Agreement, including any question regarding its existence, validity or termination, shall be referred to and finally resolved by arbitration administered by the Singapore International Arbitration Centre (“SIAC”) in accordance with the Arbitration Rules of the Singapore International Arbitration Centre (“SIAC Rules”) for the time being in force, which rules are deemed to be incorporated by reference in this clause.
+
+Prohibited Uses Policy
+You agree you will not use, or allow others to use, the Models or any derivatives of the Models to:
+1. Violate any applicable federal, state, local, or international law or regulation, or infringe upon the lawful rights or interests of any third party.
+2. Assist with, engage in or otherwise support any military purpose.
+3. Exploit, harm, or attempt to exploit or harm minors in any way.
+4. Generate or disseminate false or misleading information with the intent to cause harm.
+5. Generate or disseminate content prohibited by applicable laws or regulations.
+6. Generate or disseminate personally identifiable information without proper authorization or for unlawful or unreasonable purposes.
+7. Defame, disparage, harass, or cause harm to any individual or entity. 
+8. Conduct fully automated decision-making that adversely affects an individual’s legal rights or creates or modifies a binding, enforceable obligation.
+9. Promote discrimination, hate speech, or harmful behavior against individuals or groups based on race or ethnic origin, religion, disability, age, nationality and national origin, veteran status, sexual orientation, gender or gender identity, caste, immigration status, or any other characteristic that is associated with systemic discrimination or marginalization.

MiniMax-Text-01_Function_Call_Guide.md

@@ -0,0 +1,335 @@
+# MiniMax-Text-01 Function Call Guide
+
+## 📖 Introduction
+
+MiniMax-Text-01 model supports function calling capability, allowing the model to identify when an external function needs to be called and output function call parameters in a structured format. This document provides detailed instructions on how to use the function calling feature of MiniMax-Text-01.
+
+## 🛠️ Defining Function Calls
+
+### Function Structure
+
+Function calls need to be defined in the `tools` field of the request body. Each function consists of:
+
+```json
+{
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "function_name",           // Function name, required
+        "description": "function_description", // Brief description of the function's purpose
+        "parameters": {                    // Parameter definition in JSON Schema format
+          "type": "object",                // Overall type, fixed as "object"
+          "properties": {                  // Parameter property object
+            "param_name": {                // Parameter name
+              "description": "Parameter description",    // Description
+              "type": "string|number|boolean|array|object" // Type
+            }
+          },
+          "required": ["param1", "param2"]  // List of required parameters
+        }
+      }
+    }
+  ]
+}
+```
+
+### Example
+
+Below is a simple example of a weather query function definition:
+
+```json
+"tools": [
+  {
+    "type": "function",
+    "function": {
+      "name": "get_current_weather",
+      "description": "Get the latest weather for a location",
+      "parameters": {
+        "type": "object", 
+        "properties": {
+          "location": {
+            "type": "string", 
+            "description": "A certain city, such as Beijing, Shanghai"
+          }
+        }, 
+        "required": ["location"]
+      }
+    }
+  }
+]
+```
+
+### Complete Request Example
+
+Below is a complete Python code example that includes function definitions:
+
+```python
+payload = json.dumps({
+    "model": "MiniMax-Text-01",
+    "messages": [
+        {
+            "role": "system",
+            "content": "MM Intelligent Assistant is a large-scale language model developed by MiniMax and has no interfaces to call other products. MiniMax is a China technology company that has been committed to conducting research related to large models."
+        },
+        {
+            "role": "user",
+            "content": "What's the weather like in Shanghai today?"
+        }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_current_weather",
+                "description": "Get the latest weather for a location",
+                "parameters": {
+                  "type": "object", 
+                  "properties": {
+                    "location": {
+                      "type": "string", 
+                      "description": "A certain city, such as Beijing, Shanghai"
+                    }
+                  }, 
+                  "required": ["location"]
+                }
+            }
+        }
+    ],
+    "tool_choice": "auto",
+    "stream": True,
+    "max_tokens": 10000,
+    "temperature": 0.9,
+    "top_p": 1
+})
+```
+
+## 🔄 Function Call Input Format
+
+When processed internally by the model, function definitions are converted to a special format and concatenated to the input text:
+
+```
+<beginning_of_sentence>system function_setting=functions
+{"name": "get_current_weather", "description": "Get the latest weather for a location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "A certain city, such as Beijing, Shanghai"}}, "required": ["location"]}}<end_of_sentence>
+```
+
+Important notes:
+1. Function definitions are placed after the system settings and before the conversation data
+2. Function definitions are marked with `function_setting=functions`
+3. Each function is defined as a JSON string
+4. The area ends with `<end_of_sentence>`
+
+## 📤 Model Function Call Output
+
+When the model decides to call a function, it outputs the function call information in a special format:
+
+````
+<function_call>```typescript
+functions.get_current_weather({"location": "Shanghai"})
+```
+````
+
+"<function_call>" is a special token, followed by "functions.function_name(parameter json structure)". The parameters need to be string-matched and executed externally.
+
+## 📥 Handling Function Results
+
+After a function is successfully executed, the model will return output in the following format:
+
+````typescript
+```typescript
+functions.get_current_weather({"location": "Shanghai"})
+```
+````
+
+You can use the following regular expression method to extract the function name and parameters for subsequent processing:
+
+````python
+def parse_function_calls(content: str):
+    """
+    Parse the function call content returned by the model, extract function name and parameters
+    
+    Parameters:
+        content: The original content string returned by the model
+        
+    Returns:
+        A dictionary of parsed function call information, including function name and parameters
+    """
+    # Match typescript code block
+    pattern = r"```typescript\n(.+?)?\n```"
+    matches = re.finditer(pattern, content, re.DOTALL)
+    
+    for match in matches:
+        function_code = match.group(1)
+        # Extract function name and parameters
+        function_match = re.search(r'functions\.(\w+)\((.+)\)', function_code)
+        
+        if not function_match:
+            continue
+            
+        function_name = function_match.group(1)
+        arguments_str = function_match.group(2)
+        
+        try:
+            # Parse parameter JSON
+            arguments = json.loads(arguments_str)
+            print(f"Function call: {function_name}, Parameters: {arguments}")
+            
+            # Example: Handle weather query function
+            if function_name == "get_current_weather":
+                location = arguments.get("location", "Unknown location")
+                # Build function execution result
+                return {
+                    "role": "function", 
+                    "name": function_name, 
+                    "text": json.dumps({
+                        "location": location, 
+                        "temperature": "25", 
+                        "unit": "celsius", 
+                        "weather": "Sunny"
+                    }, ensure_ascii=False)
+                }
+        except json.JSONDecodeError as e:
+            print(f"Parameter parsing failed: {arguments_str}, Error: {e}")
+    
+    return {}
+````
+
+After successfully parsing the function call, you should add the function execution result to the conversation history so that the model can access and utilize this information in subsequent interactions.
+
+## 💻 Function Call Example with Transformers Library
+
+The official MiniMax-Text-01 repository provides a complete example of function calling using the Transformers library. You can view the source code in the [MiniMaxAI/MiniMax-Text-01 huggingface repository](https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/main.py).
+
+The following is the key part of implementing function calls using the Transformers library:
+
+```python
+def get_default_tools():
+    return [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_current_weather",
+                "description": "Get the latest weather for a location",
+                "parameters": {
+                    "type": "object", 
+                    "properties": {
+                        "location": {
+                            "type": "string", 
+                            "description": "A certain city, such as Beijing, Shanghai"
+                        }
+                    }, 
+                    "required": ["location"]
+                }
+            }
+        }
+    ]
+
+# Load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+prompt = "What's the weather like in Shanghai today?"
+messages = [
+    {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant created by Minimax based on MiniMax-Text-01 model."}]},
+    {"role": "user", "content": [{"type": "text", "text": prompt}]},
+]
+
+# Enable function call tools
+tools = get_default_tools()
+
+# Apply chat template and add tool definitions
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True,
+    tools=tools
+)
+
+# Generate response
+model_inputs = tokenizer(text, return_tensors="pt").to("cuda")
+quantized_model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype="bfloat16",
+    device_map=device_map,
+    quantization_config=quantization_config,
+    trust_remote_code=True,
+    offload_buffers=True,
+)
+generation_config = GenerationConfig(
+    max_new_tokens=20,
+    eos_token_id=200020,
+    use_cache=True,
+)
+
+# Execute generation
+generated_ids = quantized_model.generate(**model_inputs, generation_config=generation_config)
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+```
+
+### Running the Example
+
+You can run the example code using the following command:
+
+```bash
+export SAFETENSORS_FAST_GPU=1
+python main.py --quant_type int8 --world_size 8 --model_id <model_path> --enable_tools
+```
+
+Parameter description:
+- `--quant_type`: Quantization type, options are "default" or "int8"
+- `--world_size`: Number of GPUs, int8 quantization requires at least 8 GPUs
+- `--model_id`: Model path
+- `--enable_tools`: Enable function call feature
+
+### Result Processing
+As expected, you will get the following output:
+
+````base
+```typescript
+functions.get_current_weather({"location": "Shanghai"})
+```
+````
+
+You can use regular expressions to extract the function to call and its corresponding parameters:
+
+````python
+def try_parse_tool_calls(content: str):
+    pattern = r"```typescript\n(.+?)?\n```"
+    matches = re.finditer(pattern, content, re.DOTALL)
+    
+    for match in matches:
+        function_code = match.group(1)
+        function_match = re.search(r'functions\.(\w+)\((.+)\)', function_code)
+        
+        if not function_match:
+            continue
+            
+        function_name = function_match.group(1)
+        arguments_str = function_match.group(2)
+        
+        try:
+            arguments = json.loads(arguments_str)
+            print(f"tool_calls: [{{'type': 'function', 'function': {{'name': '{function_name}', 'arguments': {arguments}}}}}]")
+            
+            if function_name == "get_current_weather":
+                location = arguments.get("location", "Unknown")
+                return {"role": "function", "name": function_name, "text": f'{{"location": "{location}", "temperature": "25", "unit": "celsius", "weather": "Sun"}}'}
+        except json.JSONDecodeError as e:
+            print(f"Failed parse tools: {arguments_str}, Error: {e}")
+    
+    return {}
+````
+
+### Chat Template
+
+MiniMax-Text-01 uses a specific chat template format to process function calls. The chat template is defined in `tokenizer_config.json`:
+
+```json
+ "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<beginning_of_sentence>system ai_setting=assistant\\n' + message['content'][0]['text'] + '<end_of_sentence>\\n'}}{% elif message['role'] == 'user' %}{{ '<beginning_of_sentence>user name=user\\n' + message['content'][0]['text'] + '<end_of_sentence>\\n'}}{% elif message['role'] == 'assistant' %}{{ '<beginning_of_sentence>ai name=assistant\\n' }}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] }}{% endgeneration %}{% endfor %}{{ '<end_of_sentence>\\n' }}{% elif message['role'] == 'function' %}{{ '<beginning_of_sentence>system function_response=functions\\n' + '{\"name\": \"' + message['name'] + '\", \"response\": ' + message['content'][0]['text'] + '}' + '<end_of_sentence>\\n'}}{% endif %}{% endfor %}{% if tools %}{% for function in tools %}{{ '<beginning_of_sentence>system function_setting=functions\\n' + function | tojson + '<end_of_sentence>\\n'}}{% endfor %}{% endif %}{% if add_generation_prompt %}{{ '<beginning_of_sentence>ai name=assistant\\n' }}{% endif %}"
+```
+
+## 📝 Important Notes
+
+1. Function names should follow programming language naming conventions and avoid special characters
+2. Parameter descriptions should be concise and help the model understand the parameter's purpose and constraints
+3. The model does not guarantee that it will call a function; this depends on the user's input and the model's judgment
+4. Function results should be returned in a structured format for easy processing by the model
+5. The model might not call a function even if one is provided, depending on whether it determines a function call is appropriate for the given user query 

MiniMax-Text-01_Function_Call_Guide_CN.md

@@ -0,0 +1,335 @@
+# MiniMax-Text-01 函数调用（Function Call）功能指南
+
+## 📖 简介
+
+MiniMax-Text-01 模型支持函数调用功能，使模型能够识别何时需要调用外部函数，并以结构化格式输出函数调用参数。本文档详细介绍了如何使用 MiniMax-Text-01 的函数调用功能。
+
+## 🛠️ 函数调用的定义
+
+### 函数结构体
+
+函数调用需要在请求体中定义 `tools` 字段，每个函数由以下部分组成：
+
+```json
+{
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "function_name",           // 函数名称，必填
+        "description": "function_description", // 函数描述，应简明扼要说明函数功能
+        "parameters": {                    // 函数参数定义，符合 JSON Schema 格式
+          "type": "object",                // 参数整体类型，固定为object
+          "properties": {                  // 参数属性对象
+            "param_name": {                // 参数名称
+              "description": "参数描述",    // 参数说明
+              "type": "string|number|boolean|array|object" // 参数类型
+            }
+          },
+          "required": ["param1", "param2"]  // 必填参数列表
+        }
+      }
+    }
+  ]
+}
+```
+
+### 示例
+
+以下是一个简单的天气查询函数定义示例：
+
+```json
+"tools": [
+  {
+    "type": "function",
+    "function": {
+      "name": "get_current_weather",
+      "description": "Get the latest weather for a location",
+      "parameters": {
+        "type": "object", 
+        "properties": {
+          "location": {
+            "type": "string", 
+            "description": "A certain city, such as Beijing, Shanghai"
+          }
+        }, 
+        "required": ["location"]
+      }
+    }
+  }
+]
+```
+
+### 完整请求示例
+
+下面是一个包含函数定义的完整Python代码示例：
+
+```python
+payload = json.dumps({
+    "model": "MiniMax-Text-01",
+    "messages": [
+        {
+            "role": "system",
+            "content": "MM Intelligent Assistant is a large-scale language model developed by MiniMax and has no interfaces to call other products. MiniMax is a China technology company that has been committed to conducting research related to large models."
+        },
+        {
+            "role": "user",
+            "content": "上海今天天气怎么样？"
+        }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_current_weather",
+                "description": "Get the latest weather for a location",
+                "parameters": {
+                  "type": "object", 
+                  "properties": {
+                    "location": {
+                      "type": "string", 
+                      "description": "A certain city, such as Beijing, Shanghai"
+                    }
+                  }, 
+                  "required": ["location"]
+                }
+            }
+        }
+    ],
+    "tool_choice": "auto",
+    "stream": True,
+    "max_tokens": 10000,
+    "temperature": 0.9,
+    "top_p": 1
+})
+```
+
+## 🔄 函数调用的输入格式
+
+在模型内部处理时，函数定义会被转换为特殊格式并拼接到输入文本中：
+
+```
+<beginning_of_sentence>system function_setting=functions
+{"name": "get_current_weather", "description": "Get the latest weather for a location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "A certain city, such as Beijing, Shanghai"}}, "required": ["location"]}}<end_of_sentence>
+```
+
+注意事项：
+1. 函数定义位于系统设置之后、对话数据之前
+2. 使用 `function_setting=functions` 标记函数定义区域
+3. 每个函数定义使用JSON字符串表示
+4. 区域以 `<end_of_sentence>` 结束
+
+## 📤 模型的函数调用输出
+
+当模型决定调用函数时，它会在响应中使用特殊格式输出函数调用信息：
+
+````
+<function_call>```typescript
+functions.get_current_weather({"location": "上海"})
+```
+````
+
+"<function_call>" 是 special token, 后面的 "functions.函数名(参数 json 结构体)", 需要字符串匹配出参数, 交外部执行.
+
+## 📥 函数执行结果的处理
+
+当函数调用成功执行后，模型将返回以下格式的输出：
+
+````typescript
+```typescript
+functions.get_current_weather({"location": "Shanghai"})
+```
+````
+
+您可以使用以下正则表达式方法提取函数名称和参数，便于后续处理：
+
+````python
+def parse_function_calls(content: str):
+    """
+    解析模型返回的函数调用内容，提取函数名和参数
+    
+    参数:
+        content: 模型返回的原始内容字符串
+        
+    返回:
+        解析后的函数调用信息字典，包含函数名和参数
+    """
+    # 匹配 typescript 代码块
+    pattern = r"```typescript\n(.+?)?\n```"
+    matches = re.finditer(pattern, content, re.DOTALL)
+    
+    for match in matches:
+        function_code = match.group(1)
+        # 提取函数名和参数
+        function_match = re.search(r'functions\.(\w+)\((.+)\)', function_code)
+        
+        if not function_match:
+            continue
+            
+        function_name = function_match.group(1)
+        arguments_str = function_match.group(2)
+        
+        try:
+            # 解析参数JSON
+            arguments = json.loads(arguments_str)
+            print(f"调用函数: {function_name}, 参数: {arguments}")
+            
+            # 示例: 处理天气查询函数
+            if function_name == "get_current_weather":
+                location = arguments.get("location", "未知位置")
+                # 构建函数执行结果
+                return {
+                    "role": "function", 
+                    "name": function_name, 
+                    "text": json.dumps({
+                        "location": location, 
+                        "temperature": "25", 
+                        "unit": "celsius", 
+                        "weather": "晴朗"
+                    }, ensure_ascii=False)
+                }
+        except json.JSONDecodeError as e:
+            print(f"参数解析失败: {arguments_str}, 错误: {e}")
+    
+    return {}
+````
+
+成功解析函数调用后，您应将函数执行结果添加到对话历史中，以便模型在后续交互中能够访问和利用这些信息。
+
+## 💻 使用 Transformers 库的函数调用示例
+
+MiniMax-Text-01 官方仓库提供了使用 Transformers 库进行函数调用的完整示例。您可以在 [MiniMaxAI/MiniMax-Text-01 huggingface 仓库](https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/main.py) 中查看源代码。
+
+以下是使用 Transformers 库实现函数调用的关键部分：
+
+```python
+def get_default_tools():
+    return [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_current_weather",
+                "description": "Get the latest weather for a location",
+                "parameters": {
+                    "type": "object", 
+                    "properties": {
+                        "location": {
+                            "type": "string", 
+                            "description": "A certain city, such as Beijing, Shanghai"
+                        }
+                    }, 
+                    "required": ["location"]
+                }
+            }
+        }
+    ]
+
+# 加载模型和分词器
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+prompt = "What's the weather like in Shanghai today?"
+messages = [
+    {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant created by Minimax based on MiniMax-Text-01 model."}]},
+    {"role": "user", "content": [{"type": "text", "text": prompt}]},
+]
+
+# 启用函数调用工具
+tools = get_default_tools()
+
+# 应用聊天模板，并加入工具定义
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True,
+    tools=tools
+)
+
+# 生成回复
+model_inputs = tokenizer(text, return_tensors="pt").to("cuda")
+quantized_model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype="bfloat16",
+    device_map=device_map,
+    quantization_config=quantization_config,
+    trust_remote_code=True,
+    offload_buffers=True,
+)
+generation_config = GenerationConfig(
+    max_new_tokens=20,
+    eos_token_id=200020,
+    use_cache=True,
+)
+
+# 执行生成
+generated_ids = quantized_model.generate(**model_inputs, generation_config=generation_config)
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+```
+
+### 运行方式
+
+您可以通过以下命令运行示例代码：
+
+```bash
+export SAFETENSORS_FAST_GPU=1
+python main.py --quant_type int8 --world_size 8 --model_id <model_path> --enable_tools
+```
+
+参数说明：
+- `--quant_type`: 量化类型，可选 "default" 或 "int8"
+- `--world_size`: GPU 数量，int8 量化至少需要 8 个 GPU
+- `--model_id`: 模型路径
+- `--enable_tools`: 启用函数调用功能
+
+### 结果处理
+符合预期的情况下，你将得到以下输出
+
+````base
+```typescript
+functions.get_current_weather({"location": "Shanghai"})
+```
+````
+
+你可以使用正则表达式提取出需要调用的 function 和 对应的参数
+
+````python
+def try_parse_tool_calls(content: str):
+    pattern = r"```typescript\n(.+?)?\n```"
+    matches = re.finditer(pattern, content, re.DOTALL)
+    
+    for match in matches:
+        function_code = match.group(1)
+        function_match = re.search(r'functions\.(\w+)\((.+)\)', function_code)
+        
+        if not function_match:
+            continue
+            
+        function_name = function_match.group(1)
+        arguments_str = function_match.group(2)
+        
+        try:
+            arguments = json.loads(arguments_str)
+            print(f"tool_calls: [{{'type': 'function', 'function': {{'name': '{function_name}', 'arguments': {arguments}}}}}]")
+            
+            if function_name == "get_current_weather":
+                location = arguments.get("location", "Unknown")
+                return {"role": "function", "name": function_name, "text": f'{{"location": "{location}", "temperature": "25", "unit": "celsius", "weather": "Sun"}}'}
+        except json.JSONDecodeError as e:
+            print(f"Failed parse tools: {arguments_str}, Error: {e}")
+    
+    return {}
+````
+
+### 聊天模板
+
+MiniMax-Text-01 使用特定的聊天模板格式处理函数调用。聊天模板定义在 `tokenizer_config.json` 中：
+
+```json
+ "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<beginning_of_sentence>system ai_setting=assistant\\n' + message['content'][0]['text'] + '<end_of_sentence>\\n'}}{% elif message['role'] == 'user' %}{{ '<beginning_of_sentence>user name=user\\n' + message['content'][0]['text'] + '<end_of_sentence>\\n'}}{% elif message['role'] == 'assistant' %}{{ '<beginning_of_sentence>ai name=assistant\\n' }}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] }}{% endgeneration %}{% endfor %}{{ '<end_of_sentence>\\n' }}{% elif message['role'] == 'function' %}{{ '<beginning_of_sentence>system function_response=functions\\n' + '{\"name\": \"' + message['name'] + '\", \"response\": ' + message['content'][0]['text'] + '}' + '<end_of_sentence>\\n'}}{% endif %}{% endfor %}{% if tools %}{% for function in tools %}{{ '<beginning_of_sentence>system function_setting=functions\\n' + function | tojson + '<end_of_sentence>\\n'}}{% endfor %}{% endif %}{% if add_generation_prompt %}{{ '<beginning_of_sentence>ai name=assistant\\n' }}{% endif %}"
+
+```
+
+## 📝 注意事项
+
+1. 函数名称应当遵循编程语言的命名规范，避免使用特殊字符
+2. 参数描述应当简洁明了，帮助模型理解参数的用途和约束
+3. 模型并不保证每次都会调用函数，这取决于用户的输入和模型的判断
+4. 函数调用结果应当以结构化方式返回，便于模型理解和处理

README.md

@@ -0,0 +1,269 @@
+---
+pipeline_tag: text-generation
+---
+<div align="center">
+
+<svg width="60%" height="auto" viewBox="0 0 144 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M26.6782 7.96523C26.6782 7.02436 25.913 6.26087 24.9739 6.26087C24.0348 6.26087 23.2695 7.0261 23.2695 7.96523V36.2139C23.2695 38.4 21.4904 40.1791 19.3043 40.1791C17.1183 40.1791 15.3391 38.4 15.3391 36.2139V18.0904C15.3391 17.1496 14.5739 16.3861 13.6348 16.3861C12.6956 16.3861 11.9304 17.1513 11.9304 18.0904V25.7722C11.9304 27.9583 10.1513 29.7374 7.96518 29.7374C5.7791 29.7374 4 27.9583 4 25.7722V22.9878C4 22.3635 4.50609 21.8574 5.13043 21.8574C5.75478 21.8574 6.26087 22.3635 6.26087 22.9878V25.7722C6.26087 26.713 7.02605 27.4765 7.96518 27.4765C8.90431 27.4765 9.66954 26.7113 9.66954 25.7722V18.0904C9.66954 15.9044 11.4487 14.1252 13.6348 14.1252C15.8209 14.1252 17.6 15.9044 17.6 18.0904V36.2139C17.6 37.1548 18.3652 37.9183 19.3043 37.9183C20.2435 37.9183 21.0087 37.153 21.0087 36.2139V25.1322V7.96523C21.0087 5.77914 22.7878 4 24.9739 4C27.16 4 28.9391 5.77914 28.9391 7.96523V31.3565C28.9391 31.9809 28.433 32.487 27.8087 32.487C27.1843 32.487 26.6782 31.9809 26.6782 31.3565V7.96523ZM47.6539 14.1252C45.4678 14.1252 43.6887 15.9044 43.6887 18.0904V33.2296C43.6887 34.1704 42.9235 34.9339 41.9843 34.9339C41.0452 34.9339 40.28 34.1687 40.28 33.2296V7.96523C40.28 5.77914 38.5008 4 36.3148 4C34.1287 4 32.3496 5.77914 32.3496 7.96523V40.0348C32.3496 40.9756 31.5843 41.7391 30.6452 41.7391C29.7061 41.7391 28.9409 40.9739 28.9409 40.0348V36.0643C28.9409 35.44 28.4348 34.9339 27.8104 34.9339C27.1861 34.9339 26.68 35.44 26.68 36.0643V40.0348C26.68 42.2209 28.4591 44 30.6452 44C32.8313 44 34.6104 42.2209 34.6104 40.0348V7.96523C34.6104 7.02436 35.3756 6.26087 36.3148 6.26087C37.2539 6.26087 38.0191 7.0261 38.0191 7.96523V33.2296C38.0191 35.4156 39.7982 37.1948 41.9843 37.1948C44.1704 37.1948 45.9496 35.4156 45.9496 33.2296V18.0904C45.9496 17.1496 46.7148 16.3861 47.6539 16.3861C48.593 16.3861 49.3582 17.1513 49.3582 18.0904V31.3565C49.3582 31.9809 49.8643 32.487 50.4887 32.487C51.113 32.487 51.6191 31.9809 51.6191 31.3565V18.0904C51.6191 15.9044 49.84 14.1252 47.6539 14.1252Z" fill="url(#paint0_linear_17_483)"/>
+<path d="M68.7671 16.5615H71.2541C71.3254 16.5615 71.3845 16.5859 71.435 16.6363C71.4836 16.6868 71.5097 16.7459 71.5097 16.8172V31.1824C71.5097 31.2537 71.4854 31.3128 71.435 31.3633C71.3845 31.4137 71.3254 31.4381 71.2541 31.4381H68.7671C68.6958 31.4381 68.6367 31.4137 68.5862 31.3633C68.5358 31.3146 68.5115 31.2537 68.5115 31.1824V21.812C68.5115 21.7563 68.4976 21.7268 68.4697 21.7268C68.4419 21.7268 68.4123 21.7476 68.3845 21.7911L66.1323 25.318C66.061 25.4311 65.9619 25.4885 65.8349 25.4885H64.581C64.4541 25.4885 64.3549 25.4328 64.2836 25.318L62.0315 21.7911C62.0036 21.7494 61.9741 21.7302 61.9462 21.7372C61.9184 21.7441 61.9045 21.7772 61.9045 21.8328V31.1824C61.9045 31.2537 61.8802 31.3128 61.8297 31.3633C61.7793 31.4137 61.7202 31.4381 61.6489 31.4381H59.1619C59.0906 31.4381 59.0315 31.4137 58.981 31.3633C58.9306 31.3146 58.9062 31.2537 58.9062 31.1824V16.8172C58.9062 16.7459 58.9306 16.6868 58.981 16.6363C59.0315 16.5859 59.0906 16.5615 59.1619 16.5615H61.6489C61.7758 16.5615 61.8749 16.6189 61.9462 16.732L65.1341 21.6833C65.1758 21.7685 65.2193 21.7685 65.261 21.6833L68.4697 16.732C68.541 16.6189 68.6402 16.5615 68.7671 16.5615Z" fill="currentColor"/>
+<path d="M74.1764 31.3633C74.1259 31.3146 74.1016 31.2537 74.1016 31.1824V16.8172C74.1016 16.7459 74.1259 16.6868 74.1764 16.6363C74.2268 16.5859 74.2859 16.5615 74.3572 16.5615H76.8442C76.9155 16.5615 76.9746 16.5859 77.0251 16.6363C77.0737 16.6868 77.0998 16.7459 77.0998 16.8172V31.1824C77.0998 31.2537 77.0755 31.3128 77.0251 31.3633C76.9746 31.4137 76.9155 31.4381 76.8442 31.4381H74.3572C74.2859 31.4381 74.2268 31.4137 74.1764 31.3633Z" fill="currentColor"/>
+<path d="M88.3066 16.6361C88.3553 16.5874 88.4162 16.5613 88.4875 16.5613H90.9744C91.0457 16.5613 91.1049 16.5857 91.1553 16.6361C91.204 16.6865 91.2301 16.7457 91.2301 16.817V31.1822C91.2301 31.2535 91.2057 31.3126 91.1553 31.363C91.1049 31.4135 91.0457 31.4378 90.9744 31.4378H88.5727C88.4301 31.4378 88.331 31.3822 88.2753 31.2674L82.771 22.1717C82.7431 22.13 82.7136 22.1109 82.6858 22.1178C82.6579 22.1248 82.644 22.1578 82.644 22.2135L82.6858 31.1805C82.6858 31.2518 82.6614 31.3109 82.611 31.3613C82.5606 31.4117 82.5014 31.4361 82.4301 31.4361H79.9431C79.8718 31.4361 79.8127 31.4117 79.7623 31.3613C79.7118 31.3126 79.6875 31.2518 79.6875 31.1805V16.8152C79.6875 16.7439 79.7118 16.6848 79.7623 16.6344C79.8127 16.5839 79.8718 16.5596 79.9431 16.5596H82.3449C82.4858 16.5596 82.5849 16.617 82.6423 16.73L88.124 25.7822C88.1518 25.8239 88.1797 25.8431 88.2092 25.8361C88.2371 25.8292 88.251 25.7978 88.251 25.7404L88.2301 16.8152C88.2301 16.7439 88.2545 16.6848 88.3049 16.6344L88.3066 16.6361Z" fill="currentColor"/>
+<path d="M93.8951 31.3633C93.8446 31.3146 93.8203 31.2537 93.8203 31.1824V16.8172C93.8203 16.7459 93.8446 16.6868 93.8951 16.6363C93.9455 16.5859 94.0047 16.5615 94.076 16.5615H96.5629C96.6342 16.5615 96.6934 16.5859 96.7438 16.6363C96.7925 16.6868 96.8186 16.7459 96.8186 16.8172V31.1824C96.8186 31.2537 96.7942 31.3128 96.7438 31.3633C96.6934 31.4137 96.6342 31.4381 96.5629 31.4381H94.076C94.0047 31.4381 93.9455 31.4137 93.8951 31.3633Z" fill="currentColor"/>
+<path d="M109.267 16.5615H111.754C111.825 16.5615 111.885 16.5859 111.935 16.6363C111.984 16.6868 112.01 16.7459 112.01 16.8172V31.1824C112.01 31.2537 111.985 31.3128 111.935 31.3633C111.885 31.4137 111.825 31.4381 111.754 31.4381H109.267C109.196 31.4381 109.137 31.4137 109.086 31.3633C109.036 31.3146 109.011 31.2537 109.011 31.1824V21.812C109.011 21.7563 108.998 21.7268 108.97 21.7268C108.942 21.7268 108.912 21.7476 108.885 21.7911L106.632 25.318C106.561 25.4311 106.462 25.4885 106.335 25.4885H105.081C104.954 25.4885 104.855 25.4328 104.784 25.318L102.531 21.7911C102.504 21.7494 102.474 21.7302 102.446 21.7372C102.418 21.7441 102.405 21.7772 102.405 21.8328V31.1824C102.405 31.2537 102.38 31.3128 102.33 31.3633C102.279 31.4137 102.22 31.4381 102.149 31.4381H99.6619C99.5906 31.4381 99.5315 31.4137 99.481 31.3633C99.4306 31.3146 99.4062 31.2537 99.4062 31.1824V16.8172C99.4062 16.7459 99.4306 16.6868 99.481 16.6363C99.5315 16.5859 99.5906 16.5615 99.6619 16.5615H102.149C102.276 16.5615 102.375 16.6189 102.446 16.732L105.634 21.6833C105.676 21.7685 105.719 21.7685 105.761 21.6833L108.97 16.732C109.041 16.6189 109.14 16.5615 109.267 16.5615Z" fill="currentColor"/>
+<path d="M123.782 31.2241L123.144 29.1424C123.116 29.0867 123.079 29.0572 123.038 29.0572H117.81C117.768 29.0572 117.732 29.085 117.704 29.1424L117.088 31.2241C117.046 31.3668 116.954 31.4363 116.812 31.4363H114.112C114.027 31.4363 113.963 31.412 113.921 31.3615C113.879 31.3128 113.871 31.2381 113.9 31.1389L118.49 16.7737C118.532 16.6328 118.624 16.5615 118.766 16.5615H122.102C122.243 16.5615 122.335 16.6328 122.379 16.7737L126.968 31.1389C126.982 31.1668 126.989 31.2033 126.989 31.245C126.989 31.372 126.911 31.4363 126.756 31.4363H124.057C123.916 31.4363 123.824 31.365 123.78 31.2241H123.782ZM118.554 26.7407H122.295C122.38 26.7407 122.408 26.6989 122.38 26.6137L120.467 20.3024C120.453 20.2467 120.432 20.2207 120.403 20.2276C120.375 20.2346 120.352 20.2589 120.339 20.3024L118.469 26.6137C118.455 26.6989 118.483 26.7407 118.554 26.7407Z" fill="currentColor"/>
+<path d="M128.222 31.353C128.18 31.2974 128.187 31.2261 128.243 31.1409L132.365 24.0643C132.393 24.0226 132.393 23.9791 132.365 23.9374L128.243 16.8609L128.201 16.7339C128.201 16.6209 128.28 16.5635 128.434 16.5635H131.133C131.274 16.5635 131.38 16.6209 131.452 16.7339L134.213 21.6C134.255 21.6852 134.299 21.6852 134.34 21.6L137.102 16.7339C137.173 16.6209 137.28 16.5635 137.42 16.5635H140.099C140.198 16.5635 140.269 16.5913 140.311 16.6487C140.353 16.7061 140.346 16.7756 140.29 16.8609L136.168 23.9374C136.154 23.9791 136.154 24.0226 136.168 24.0643L140.29 31.1409L140.332 31.2678C140.332 31.3809 140.253 31.4383 140.099 31.4383H137.42C137.278 31.4383 137.172 31.3826 137.102 31.2678L134.34 26.4226C134.299 26.3374 134.255 26.3374 134.213 26.4226L131.429 31.2678C131.358 31.3809 131.252 31.4383 131.111 31.4383H128.433C128.333 31.4383 128.262 31.4104 128.22 31.353H128.222Z" fill="currentColor"/>
+<defs>
+<linearGradient id="paint0_linear_17_483" x1="3.99826" y1="24" x2="51.6208" y2="24" gradientUnits="userSpaceOnUse">
+<stop stop-color="#E21680"/>
+<stop offset="1" stop-color="#FF633A"/>
+</linearGradient>
+</defs>
+</svg>
+
+</div>
+<hr>
+
+<div align="center" style="line-height: 1;">
+  <a href="https://www.minimax.io" target="_blank" style="margin: 2px;">
+    <img alt="Homepage" src="https://img.shields.io/badge/_Homepage-MiniMax-FF4040?style=flat-square&labelColor=2C3E50&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2aWV3Qm94PSIwIDAgNDkwLjE2IDQxMS43Ij48ZGVmcz48c3R5bGU+LmNscy0xe2ZpbGw6I2ZmZjt9PC9zdHlsZT48L2RlZnM+PHBhdGggY2xhc3M9ImNscy0xIiBkPSJNMjMzLjQ1LDQwLjgxYTE3LjU1LDE3LjU1LDAsMSwwLTM1LjEsMFYzMzEuNTZhNDAuODIsNDAuODIsMCwwLDEtODEuNjMsMFYxNDVhMTcuNTUsMTcuNTUsMCwxLDAtMzUuMDksMHY3OS4wNmE0MC44Miw0MC44MiwwLDAsMS04MS42MywwVjE5NS40MmExMS42MywxMS42MywwLDAsMSwyMy4yNiwwdjI4LjY2YTE3LjU1LDE3LjU1LDAsMCwwLDM1LjEsMFYxNDVBNDAuODIsNDAuODIsMCwwLDEsMTQwLDE0NVYzMzEuNTZhMTcuNTUsMTcuNTUsMCwwLDAsMzUuMSwwVjIxNy41aDBWNDAuODFhNDAuODEsNDAuODEsMCwxLDEsODEuNjIsMFYyODEuNTZhMTEuNjMsMTEuNjMsMCwxLDEtMjMuMjYsMFptMjE1LjksNjMuNEE0MC44Niw0MC44NiwwLDAsMCw0MDguNTMsMTQ1VjMwMC44NWExNy41NSwxNy41NSwwLDAsMS0zNS4wOSwwdi0yNjBhNDAuODIsNDAuODIsMCwwLDAtODEuNjMsMFYzNzAuODlhMTcuNTUsMTcuNTUsMCwwLDEtMzUuMSwwVjMzMGExMS42MywxMS42MywwLDEsMC0yMy4yNiwwdjQwLjg2YTQwLjgxLDQwLjgxLDAsMCwwLDgxLjYyLDBWNDAuODFhMTcuNTUsMTcuNTUsMCwwLDEsMzUuMSwwdjI2MGE0MC44Miw0MC44MiwwLDAsMCw4MS42MywwVjE0NWExNy41NSwxNy41NSwwLDEsMSwzNS4xLDBWMjgxLjU2YTExLjYzLDExLjYzLDAsMCwwLDIzLjI2LDBWMTQ1QTQwLjg1LDQwLjg1LDAsMCwwLDQ0OS4zNSwxMDQuMjFaIi8+PC9zdmc+&logoWidth=20" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+  <a href="https://arxiv.org/abs/2501.08313" target="_blank" style="margin: 2px;">
+    <img alt="Paper" src="https://img.shields.io/badge/📖_Paper-MiniMax--01-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+  <a href="https://chat.minimax.io/" target="_blank" style="margin: 2px;">
+    <img alt="Chat" src="https://img.shields.io/badge/_MiniMax_Chat-FF4040?style=flat-square&labelColor=2C3E50&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2aWV3Qm94PSIwIDAgNDkwLjE2IDQxMS43Ij48ZGVmcz48c3R5bGU+LmNscy0xe2ZpbGw6I2ZmZjt9PC9zdHlsZT48L2RlZnM+PHBhdGggY2xhc3M9ImNscy0xIiBkPSJNMjMzLjQ1LDQwLjgxYTE3LjU1LDE3LjU1LDAsMSwwLTM1LjEsMFYzMzEuNTZhNDAuODIsNDAuODIsMCwwLDEtODEuNjMsMFYxNDVhMTcuNTUsMTcuNTUsMCwxLDAtMzUuMDksMHY3OS4wNmE0MC44Miw0MC44MiwwLDAsMS04MS42MywwVjE5NS40MmExMS42MywxMS42MywwLDAsMSwyMy4yNiwwdjI4LjY2YTE3LjU1LDE3LjU1LDAsMCwwLDM1LjEsMFYxNDVBNDAuODIsNDAuODIsMCwwLDEsMTQwLDE0NVYzMzEuNTZhMTcuNTUsMTcuNTUsMCwwLDAsMzUuMSwwVjIxNy41aDBWNDAuODFhNDAuODEsNDAuODEsMCwxLDEsODEuNjIsMFYyODEuNTZhMTEuNjMsMTEuNjMsMCwxLDEtMjMuMjYsMFptMjE1LjksNjMuNEE0MC44Niw0MC44NiwwLDAsMCw0MDguNTMsMTQ1VjMwMC44NWExNy41NSwxNy41NSwwLDAsMS0zNS4wOSwwdi0yNjBhNDAuODIsNDAuODIsMCwwLDAtODEuNjMsMFYzNzAuODlhMTcuNTUsMTcuNTUsMCwwLDEtMzUuMSwwVjMzMGExMS42MywxMS42MywwLDEsMC0yMy4yNiwwdjQwLjg2YTQwLjgxLDQwLjgxLDAsMCwwLDgxLjYyLDBWNDAuODFhMTcuNTUsMTcuNTUsMCwwLDEsMzUuMSwwdjI2MGE0MC44Miw0MC44MiwwLDAsMCw4MS42MywwVjE0NWExNy41NSwxNy41NSwwLDEsMSwzNS4xLDBWMjgxLjU2YTExLjYzLDExLjYzLDAsMCwwLDIzLjI2LDBWMTQ1QTQwLjg1LDQwLjg1LDAsMCwwLDQ0OS4zNSwxMDQuMjFaIi8+PC9zdmc+&logoWidth=20" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+  <a href="https://www.minimax.io/platform" style="margin: 2px;">
+    <img alt="API" src="https://img.shields.io/badge/⚡_API-Platform-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+  <a href="https://github.com/MiniMax-AI/MiniMax-MCP" style="margin: 2px;">
+    <img alt="MCP" src="https://img.shields.io/badge/🚀_MCP-MiniMax_MCP-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+</div>
+<div align="center" style="line-height: 1;">
+  <a href="https://github.com/MiniMax-AI/MiniMax-01" target="_blank" style="margin: 2px;">
+    <img alt="GitHub" src="https://img.shields.io/badge/_GitHub-MinMax-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+  <a href="https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/LICENSE-MODEL" style="margin: 2px;">
+    <img alt="Model License" src="https://img.shields.io/badge/_Model_License-Model_Agreement-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+   <a href="https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/LICENSE-CODE" style="margin: 2px;">
+    <img alt="Code License" src="https://img.shields.io/badge/_Code_License-MIT-FF4040?style=flat-square&labelColor=2C3E50" style="display: inline-block; vertical-align: middle;"/>
+  </a>
+</div>
+<div align="center" style="line-height: 1;">
+  <a href="https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/figures/wechat-qrcode.jpeg" target="_blank" style="margin: 2px;">
+    WeChat
+  </a>
+</div>
+
+
+# MiniMax-Text-01
+
+## 1. Introduction
+
+MiniMax-Text-01 is a powerful language model with 456 billion total parameters, of which 45.9 billion are activated per token. To better unlock the long context capabilities of the model, MiniMax-Text-01 adopts a hybrid architecture that combines Lightning Attention, Softmax Attention and Mixture-of-Experts (MoE). Leveraging advanced parallel strategies and innovative compute-communication overlap methods—such as Linear Attention Sequence Parallelism Plus (LASP+), varlen ring attention, Expert Tensor Parallel (ETP), etc., MiniMax-Text-01's training context length is extended to 1 million tokens, and it can handle a context of up to 4 million tokens during the inference. On various academic benchmarks, MiniMax-Text-01 also demonstrates the performance of a top-tier model.
+
+<p align="center">
+  <img width="100%" src="figures/TextBench.png">
+</p>
+
+## 2. Model Architecture
+
+The architecture of MiniMax-Text-01 is briefly described as follows:
+- Total Parameters: 456B
+- Activated Parameters per Token: 45.9B
+- Number Layers: 80
+- Hybrid Attention: a softmax attention is positioned after every 7 lightning attention.
+  - Number of attention heads: 64
+  - Attention head dimension: 128
+- Mixture of Experts:
+  - Number of experts: 32
+  - Expert hidden dimension: 9216
+  - Top-2 routing strategy
+- Positional Encoding: Rotary Position Embedding (RoPE) applied to half of the attention head dimension with a base frequency of 10,000,000
+- Hidden Size: 6144
+- Vocab Size: 200,064
+
+## 3. Evaluation
+
+### Core Academic Benchmarks
+
+| **Tasks**                     | **GPT-4o (11-20)** | **Claude-3.5-Sonnet (10-22)** | **Gemini-1.5-Pro (002)** | **Gemini-2.0-Flash (exp)** | **Qwen2.5-72B-Inst.** | **DeepSeek-V3** | **Llama-3.1-405B-Inst.** | **MiniMax-Text-01** |
+|-------------------------------|--------------------|-------------------------------|--------------------------|----------------------------|-----------------------|-----------------|--------------------------|---------------------|
+| **General**                   |                    |                               |                          |                            |                       |                 |                          |                     |
+| MMLU<sup>*</sup>                      | 85.7               | 88.3                          | 86.8                     | 86.5                       | 86.1                  | 88.5        | **88.6**                 | 88.5                |
+| MMLU-Pro<sup>*</sup>                  | 74.4               | **78.0**                      | 75.8                     | 76.4                       | 71.1                  | 75.9            | 73.3                     | 75.7                |
+| SimpleQA                      | **39.0**           | 28.1                          | 23.4                     | 26.6                       | 10.3                  | 24.9            | 23.2                     | 23.7                |
+| C-SimpleQA                    | 64.6               | 56.8                          | 59.4                     | 63.3                       | 52.2                  | 64.8            | 54.7                     | **67.4**            |
+| IFEval _(avg)_                | 84.1               | **90.1**                      | 89.4                     | 88.4                       | 87.2                  | 87.3            | 86.4                     | 89.1                |
+| Arena-Hard                    | **92.4**           | 87.6                          | 85.3                     | 72.7                       | 81.2                  | 91.4            | 63.5                     | 89.1                |
+| **Reasoning**                 |                    |                               |                          |                            |                       |                 |                          |                     |
+| GPQA<sup>*</sup> _(diamond)_          | 46.0               | **65.0**                      | 59.1                     | 62.1                       | 49.0                  | 59.1            | 50.7                     | 54.4                |
+| DROP<sup>*</sup> _(F1)_               | 89.2               | 88.8                          | 89.2                     | 89.3                       | 85.0                  | 91.0        | **92.5**                 | 87.8                |
+| **Mathematics**               |                    |                               |                          |                            |                       |                 |                          |                     |
+| GSM8k<sup>*</sup>                     | 95.6               | **96.9**                      | 95.2                     | 95.4                       | 95.8                  | 96.7            | 96.7                     | 94.8                |
+| MATH<sup>*</sup>                      | 76.6               | 74.1                          | **84.6**                 | 83.9                       | 81.8                  | **84.6**        | 73.8                     | 77.4                |
+| **Coding**                    |                    |                               |                          |                            |                       |                 |                          |                     |
+| MBPP +                        | 76.2               | 75.1                          | 75.4                     | 75.9                       | 77.0              | **78.8**        | 73.0                     | 71.7                |
+| HumanEval                     | 90.2               | **93.7**                      | 86.6                     | 89.6                       | 86.6                  | 92.1            | 89.0                     | 86.9                |
+
+<sup>*</sup> Evaluated following a _0-shot CoT_ setting.
+
+### Long Benchmarks
+#### 4M Needle In A Haystack Test
+<p align="center">
+  <img width="90%" src="figures/niah.png">
+</p>
+
+#### Ruler
+| Model | 4k | 8k | 16k | 32k | 64k | 128k | 256k | 512k | 1M |
+|-------|----|----|-----|-----|-----|------|------|------|----|
+| **GPT-4o (11-20)** | **0.970** | 0.921 | 0.890 | 0.888 | 0.884 | - | - | - | - |
+| **Claude-3.5-Sonnet (10-22)** | 0.965 | 0.960 | 0.957 | 0.950 | **0.952** | 0.938 | - | - | - |
+| **Gemini-1.5-Pro (002)** | 0.962 | 0.960 | **0.960** | **0.958** | 0.938 | 0.917 | 0.916 | 0.861 | 0.850 |
+| **Gemini-2.0-Flash (exp)** | 0.960 | 0.960 | 0.951 | 0.957 | 0.937 | 0.860 | 0.797 | 0.709 | - |
+| **MiniMax-Text-01** | 0.963 | **0.961** | 0.953 | 0.954 | 0.943 | **0.947** | **0.945** | **0.928** | **0.910** |
+
+#### LongBench v2
+| **Model**                  | **overall** | **easy** | **hard** | **short** | **medium** | **long** |
+|----------------------------|-------------|----------|----------|------------|------------|----------|
+| Human                      | 53.7        | 100.0    | 25.1     | 47.2       | 59.1       | 53.7     |
+| **w/ CoT**                 |             |          |          |            |            |          |
+| GPT-4o (11-20)             | 51.4        | 54.2     | 49.7     | 59.6       | 48.6       | 43.5     |
+| Claude-3.5-Sonnet (10-22)  | 46.7        | 55.2     | 41.5     | 53.9       | 41.9       | 44.4     |
+| Deepseek-V3                | -           | -        | -        | -          | -          | -        |
+| Qwen2.5-72B-Inst.          | 43.5        | 47.9     | 40.8     | 48.9       | 40.9       | 39.8     |
+| **MiniMax-Text-01**        | **56.5**    | **66.1** | **50.5** | **61.7**   | **56.7**   | **47.2** |
+| **w/o CoT**                |             |          |          |            |            |          |
+| GPT-4o (11-20)             | 50.1        | 57.4     | 45.6     | 53.3       | 52.4       | 40.2     |
+| Claude-3.5-Sonnet (10-22)  | 41.0        | 46.9     | 37.3     | 46.1       | 38.6       | 37.0     |
+| Deepseek-V3                | 48.7        | -        | -        | -          | -          | -        |
+| Qwen2.5-72B-Inst.          | 42.1        | 42.7     | 41.8     | 45.6       | 38.1       | **44.4** |
+| **MiniMax-Text-01**        | **52.9**    | **60.9** | **47.9** | **58.9**   | **52.6**   | 43.5     |
+
+#### MTOB
+| **Context Type** | **no context** | **half book** | **full book** | **Δ half book** | **Δ full book** |
+|------------------|----------------|---------------|---------------|------------------|-----------------|
+| **eng → kalam (ChrF)** | | | | | |
+| GPT-4o (11-20) | 9.90 | **54.30** | - | 44.40 | - |
+| Claude-3.5-Sonnet (10-22) | 20.22 | 53.62 | 55.65 | 33.39 | 35.42 |
+| Gemini-1.5-Pro (002) | 16.79 | 53.68 | **57.90** | 36.89 | 41.11 |
+| Gemini-2.0-Flash (exp) | 12.20 | 49.50 | 53.30 | 37.30 | 41.10 |
+| Qwen-Long | 16.55 | 48.48 | 45.94 | 31.92 | 29.39 |
+| **MiniMax-Text-01** | 6.0 | 51.74 | 51.60 | **45.7** | **45.6** |
+| **kalam → eng (BLEURT)** | | | | | |
+| GPT-4o (11-20) | 33.20 | 58.30 | - | 25.10 | - |
+| Claude-3.5-Sonnet (10-22) | 31.42 | 59.70 | 62.30 | 28.28 | 30.88 |
+| Gemini-1.5-Pro (002) | 32.02 | **61.52** | **63.09** | **29.50** | **31.07** |
+| Gemini-2.0-Flash (exp) | 33.80 | 57.50 | 57.00 | 23.70 | 23.20 |
+| Qwen-Long | 30.13 | 53.14 | 32.15 | 23.01 | 2.02 |
+| **MiniMax-Text-01** | 33.65 | 57.10 | 58.00 | 23.45 | 24.35 |
+
+
+## 4. Quickstart
+Here we provide a simple example of loading the tokenizer and model to generate content.
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, QuantoConfig, GenerationConfig
+
+# load hf config
+hf_config = AutoConfig.from_pretrained("MiniMaxAI/MiniMax-Text-01", trust_remote_code=True)
+
+# quantization config, int8 is recommended
+quantization_config =  QuantoConfig(
+            weights="int8",
+            modules_to_not_convert=[
+                "lm_head",
+                "embed_tokens",
+            ] + [f"model.layers.{i}.coefficient" for i in range(hf_config.num_hidden_layers)]
+            + [f"model.layers.{i}.block_sparse_moe.gate" for i in range(hf_config.num_hidden_layers)]
+        )
+
+# assume 8 GPUs
+world_size = 8
+layers_per_device = hf_config.num_hidden_layers // world_size
+# set device map
+device_map = {
+    'model.embed_tokens': 'cuda:0',
+    'model.norm': f'cuda:{world_size - 1}',
+    'lm_head': f'cuda:{world_size - 1}'
+}
+for i in range(world_size):
+    for j in range(layers_per_device):
+        device_map[f'model.layers.{i * layers_per_device + j}'] = f'cuda:{i}'
+
+# load tokenizer
+tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01")
+prompt = "Hello!"
+messages = [
+    {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant created by MiniMax based on MiniMax-Text-01 model."}]},
+    {"role": "user", "content": [{"type": "text", "text": prompt}]},
+]
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+# tokenize and move to device
+model_inputs = tokenizer(text, return_tensors="pt").to("cuda")
+
+# load bfloat16 model, move to device, and apply quantization
+quantized_model = AutoModelForCausalLM.from_pretrained(
+    "MiniMaxAI/MiniMax-Text-01",
+    torch_dtype="bfloat16",
+    device_map=device_map,
+    quantization_config=quantization_config,
+    trust_remote_code=True,
+    offload_buffers=True,
+)
+
+# generate response
+generation_config = GenerationConfig(
+    max_new_tokens=20,
+    eos_token_id=200020,
+    use_cache=True,
+)
+generated_ids = quantized_model.generate(**model_inputs, generation_config=generation_config)
+print(f"generated_ids: {generated_ids}")
+generated_ids = [
+    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+]
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+```
+
+## 5. Deployment Guide
+For production deployment, we recommend using [vLLM](https://docs.vllm.ai/en/latest/) to serve MiniMax-Text-01. vLLM provides excellent performance for serving large language models with the following features:
+
+🔥 Outstanding service throughput performance  
+⚡ Efficient and intelligent memory management  
+📦 Powerful batch request processing capability  
+⚙️ Deeply optimized underlying performance  
+
+For detailed deployment instructions, please refer to our [vLLM Deployment Guide](https://github.com/MiniMax-AI/MiniMax-01/blob/main/docs/vllm_deployment_guide.md).
+
+## 6. Function Calling
+MiniMax-Text-01 supports Function Calling capability, enabling the model to intelligently identify when external functions need to be called and output parameters in structured JSON format. With Function Calling, you can:
+- Let the model recognize implicit function call needs in user requests
+- Receive structured parameter outputs for seamless application integration
+- Support various complex parameter types, including nested objects and arrays
+Function Calling supports standard OpenAI-compatible format definitions and integrates seamlessly with the Transformers library. For detailed usage instructions, please refer to our [Function Call Guide](./MiniMax-Text-01_Function_Call_Guide.md) or [Chinese Guide](./MiniMax-Text-01_Function_Call_Guide_CN.md).
+
+## 7. Citation
+
+```
+@misc{minimax2025minimax01scalingfoundationmodels,
+      title={MiniMax-01: Scaling Foundation Models with Lightning Attention}, 
+      author={MiniMax and Aonian Li and Bangwei Gong and Bo Yang and Boji Shan and Chang Liu and Cheng Zhu and Chunhao Zhang and Congchao Guo and Da Chen and Dong Li and Enwei Jiao and Gengxin Li and Guojun Zhang and Haohai Sun and Houze Dong and Jiadai Zhu and Jiaqi Zhuang and Jiayuan Song and Jin Zhu and Jingtao Han and Jingyang Li and Junbin Xie and Junhao Xu and Junjie Yan and Kaishun Zhang and Kecheng Xiao and Kexi Kang and Le Han and Leyang Wang and Lianfei Yu and Liheng Feng and Lin Zheng and Linbo Chai and Long Xing and Meizhi Ju and Mingyuan Chi and Mozhi Zhang and Peikai Huang and Pengcheng Niu and Pengfei Li and Pengyu Zhao and Qi Yang and Qidi Xu and Qiexiang Wang and Qin Wang and Qiuhui Li and Ruitao Leng and Shengmin Shi and Shuqi Yu and Sichen Li and Songquan Zhu and Tao Huang and Tianrun Liang and Weigao Sun and Weixuan Sun and Weiyu Cheng and Wenkai Li and Xiangjun Song and Xiao Su and Xiaodong Han and Xinjie Zhang and Xinzhu Hou and Xu Min and Xun Zou and Xuyang Shen and Yan Gong and Yingjie Zhu and Yipeng Zhou and Yiran Zhong and Yongyi Hu and Yuanxiang Fan and Yue Yu and Yufeng Yang and Yuhao Li and Yunan Huang and Yunji Li and Yunpeng Huang and Yunzhi Xu and Yuxin Mao and Zehan Li and Zekang Li and Zewei Tao and Zewen Ying and Zhaoyang Cong and Zhen Qin and Zhenhua Fan and Zhihang Yu and Zhuo Jiang and Zijia Wu},
+      year={2025},
+      eprint={2501.08313},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2501.08313}, 
+}
+```
+
+## 8. Chatbot & API
+For general use and evaluation, we provide a [Chatbot](https://chat.minimax.io/) with online search capabilities and the [online API](https://www.minimax.io/platform) for developers. For general use and evaluation, we provide the [MiniMax MCP Server](https://github.com/MiniMax-AI/MiniMax-MCP) with video generation, image generation, speech synthesis, and voice cloning for developers.
+
+## 9. Contact Us
+Contact us at [[email protected]](mailto:[email protected]).

config.json

@@ -0,0 +1,126 @@
+{
+  "architectures": [
+    "MiniMaxText01ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "layer_types": [
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_minimax_text_01.MiniMaxText01Config",
+    "AutoModelForCausalLM": "modeling_minimax_text_01.MiniMaxText01ForCausalLM"
+  },
+  "bos_token_id": null,
+  "eos_token_id": null,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 9216,
+  "layernorm_full_attention_alpha": 3.5565588200778455,
+  "layernorm_full_attention_beta": 1.0,
+  "layernorm_linear_attention_alpha": 3.5565588200778455,
+  "layernorm_linear_attention_beta": 1.0,
+  "layernorm_mlp_alpha": 3.5565588200778455,
+  "layernorm_mlp_beta": 1.0,
+  "max_position_embeddings": 10240000,
+  "model_type": "minimax_text_01",
+  "num_attention_heads": 64,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "num_local_experts": 32,
+  "output_router_logits": false,
+  "postnorm": true,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000000,
+  "rotary_dim": 64,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "shared_intermediate_size": 0,
+  "shared_moe_mode": "sigmoid",
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_size": 200064
+}

configuration_minimax_text_01.py

@@ -0,0 +1,152 @@
+""" MiniMaxText01 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class MiniMaxText01Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`MiniMaxText01Model`]. It is used to instantiate an
+    MiniMaxText01 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the MiniMaxText01.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the MiniMaxText01 model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`MiniMaxText01Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 14336):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 8):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
+            The maximum sequence length that this model might ever be used with. MiniMaxText01's sliding window attention
+            allows sequence of up to 4096*32 tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            The id of the padding token.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            The id of the "beginning-of-sequence" token.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            The id of the "end-of-sequence" token.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 1000000.0):
+            The base period of the RoPE embeddings.
+        sliding_window (`int`, *optional*):
+            Sliding window attention window size. If not specified, will default to `4096`.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        num_experts_per_tok (`int`, *optional*, defaults to 2):
+            The number of experts to route per-token, can be also interpreted as the `top-k` routing
+            parameter
+        num_local_experts (`int`, *optional*, defaults to 8):
+            Number of experts per Sparse MLP layer.
+        output_router_logits (`bool`, *optional*, defaults to `False`):
+            Whether or not the router logits should be returned by the model. Enabeling this will also
+            allow the model to output the auxiliary loss. See [here]() for more details
+        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
+            The aux loss factor for the total loss.
+        router_jitter_noise (`float`, *optional*, defaults to 0.0):
+            Amount of noise to add to the router.
+
+    ```python
+    >>> from transformers import MiniMaxText01Model, MiniMaxText01Config
+
+    >>> # Initializing a MiniMaxText01 style configuration
+    >>> configuration = MiniMaxText01Config()
+
+    >>> # Initializing a model from the MiniMaxText01 style configuration
+    >>> model = MiniMaxText01Model(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "MiniMaxText01"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=14336,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=8,
+        hidden_act="silu",
+        max_position_embeddings=4096 * 32,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=None,
+        eos_token_id=None,
+        tie_word_embeddings=False,
+        rope_theta=1e6,
+        sliding_window=None,
+        attention_dropout=0.0,
+        num_experts_per_tok=2,
+        num_local_experts=8,
+        output_router_logits=False,
+        router_aux_loss_coef=0.001,
+        router_jitter_noise=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.router_jitter_noise = router_jitter_noise
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

main.py

@@ -0,0 +1,100 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, QuantoConfig, GenerationConfig
+import torch
+import argparse
+
+"""
+ usage:
+    export SAFETENSORS_FAST_GPU=1
+    python main.py --quant_type int8 --world_size 8 --model_id <model_path>
+"""
+
+def generate_quanto_config(hf_config: AutoConfig, quant_type: str):
+    QUANT_TYPE_MAP = {
+        "default": None,
+        "int8": QuantoConfig(
+            weights="int8",
+            modules_to_not_convert=[
+                "lm_head",
+                "embed_tokens",
+            ] + [f"model.layers.{i}.coefficient" for i in range(hf_config.num_hidden_layers)]
+            + [f"model.layers.{i}.block_sparse_moe.gate" for i in range(hf_config.num_hidden_layers)]
+        ),
+    }
+    return QUANT_TYPE_MAP[quant_type]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--quant_type", type=str, default="default", choices=["default", "int8"])
+    parser.add_argument("--model_id", type=str, required=True)
+    parser.add_argument("--world_size", type=int, required=True)
+    return parser.parse_args()
+
+
+def check_params(args, hf_config: AutoConfig):
+    if args.quant_type == "int8":
+        assert args.world_size >= 8, "int8 weight-only quantization requires at least 8 GPUs"
+
+    assert hf_config.num_hidden_layers % args.world_size == 0, f"num_hidden_layers({hf_config.num_hidden_layers}) must be divisible by world_size({args.world_size})"
+
+
+@torch.no_grad()
+def main():
+    args = parse_args()
+    print("\n=============== Argument ===============")
+    for key in vars(args):
+        print(f"{key}: {vars(args)[key]}")
+    print("========================================")
+
+    model_id = args.model_id
+
+    hf_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+    check_params(args, hf_config)
+    quantization_config = generate_quanto_config(hf_config, args.quant_type)
+ 
+    device_map = {
+        'model.embed_tokens': 'cuda:0',
+        'model.norm': f'cuda:{args.world_size - 1}',
+        'lm_head': f'cuda:{args.world_size - 1}'
+    }
+    layers_per_device = hf_config.num_hidden_layers // args.world_size
+    for i in range(args.world_size):
+        for j in range(layers_per_device):
+            device_map[f'model.layers.{i * layers_per_device + j}'] = f'cuda:{i}'
+
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    prompt = "Hello!"
+    messages = [
+        {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant created by Minimax based on MiniMax-Text-01 model."}]},
+        {"role": "user", "content": [{"type": "text", "text": prompt}]},
+    ]
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer(text, return_tensors="pt").to("cuda")
+    quantized_model = AutoModelForCausalLM.from_pretrained(
+        model_id,
+        torch_dtype="bfloat16",
+        device_map=device_map,
+        quantization_config=quantization_config,
+        trust_remote_code=True,
+        offload_buffers=True,
+    )
+    generation_config = GenerationConfig(
+        max_new_tokens=20,
+        eos_token_id=200020,
+        use_cache=True,
+    )
+    generated_ids = quantized_model.generate(**model_inputs, generation_config=generation_config)
+    print(f"generated_ids: {generated_ids}")
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    print(response)
+
+if __name__ == "__main__":
+    main()
+

model-00000-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45b85926994c2ee092c2319158471ed9262d146bd5973c442e135dae9e21624d
+size 4916773000

model-00001-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cdaea944f170b60d206e41a80accbfcf5b9c74744f014a819c30f45cb9a9130
+size 2191113152

model-00002-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f15b83f1afd5c5da8853b7d9bd2c9814dbcb2de7a2f1a24765e16aa7a310d82
+size 2330307784

model-00003-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83f66465aca90c07b247e218950c1d03594dc08c7170ad2e1a8aaa82b26612fe
+size 2254810656

model-00004-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7ddaa7380c6dc2cc109b209bfb8450f56bd88f03bcba19f9f0735d24d65a1ed
+size 2116402376

model-00005-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99060d0311ad70c53858396b8fbf2a24e2476b965e809335aac99eb3a536c685
+size 2103016184

model-00006-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4785d81a7df999ea5ffd7688ddc2228c607c6b01173e6e53cbe718fa2da6f8b2
+size 2254810688

model-00007-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1306d2e054fd20b6900b6d50965ee12257116f5967747a6cfe6dcfd5d8d4f8cd
+size 2116402392

model-00008-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afb8c7602baf0a815e8ad02cbecfb80a9ae5d29f9ce10a578762544a5de1c0cb
+size 2202839784

model-00009-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1d8a6c70f49e8adaf8dafd15c8235be9331ffb975f2dce5ffe986cfca770598
+size 2151680440

model-00010-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f94ec16a2f79adf7fa7eb9ba583fe5441146a8ee9405eb3a9e4ba62ce58b0016
+size 2264926800

model-00011-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4bd5579bb8314355e60ab945da0d43332914854a6877e1c7e29c0796bdcc45
+size 2151680448

model-00012-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad1363efef21cc1eff543eb1e18cdc35b706925468b31746589dd5520b27c44
+size 2264926776

model-00013-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4ed9d125a753491a1a52257e4c3313c81e4b23ceec1ea92060f21fda6506d33
+size 2151680456

model-00014-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3874e30152a24167cffb45a368d644e848b39191a18250c8baa372badf1404fa
+size 2264926792

model-00015-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33fa7b68a1ca03cd36dc34b43098a4342cb90b6ab29a4ed81bbc5fd22ff3206b
+size 2151680440

model-00016-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52a2cdc9917920ae2e678dd8c8f5118bd269310dc24e09a9795def0bfc6db289
+size 2264926792

model-00017-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:966bb346a62cb5bcaf692cae407fb53e3ba58a6a7798a5012e7979110d44debb
+size 2151680456

model-00018-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6acadaecce036887bbf3d9e80bd9765f3f63ed147b0318c47dc151c484ee5dea
+size 2264926776

model-00019-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d45b4bfc0d98ffea2bbfa70672199845c424645139170705d1a615cb4b32bedf
+size 2151680456

model-00020-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d89e6f890817b4eaacbb61d447ea9741a1ae1750bb6b555f49f97eb53eed3a
+size 2264926792

model-00021-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:989d3448acab544b95096da7cb3fa2530c658c54eb9a067ef0f9735d0fed98e9
+size 2151680440

model-00022-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4079b9211347c16ab94cf92b3be67b2506f7fd22924feeb886c511878ada34a
+size 2264926792

model-00023-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa5667b97d73be5c951a84fd5141d7e5098854e386b08de3ad2d4a26619121d0
+size 2151680456

model-00024-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1b4cfc292b3496550591242fa27d8853f48f26952a68a62264559e2a8a7026c
+size 2264926776

model-00025-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0084a9a0f19e13b06b7fd6c2ab77212ecabc1237991715a4cc0a8b3760bab059
+size 2151680456

model-00026-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec99d07905384e93ff617910268fd614403273bb72a0e9a66e23477609404d39
+size 2264926800

model-00027-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf1ae16fe8da6e760ccdcbff16b81047460783c12a97e35572db64b705832b36
+size 2151680440

model-00028-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce157751c2a5278886267e41e2ad71a8fdc7bdd5f6fa798c939234992dd11ab2
+size 2264926792

model-00029-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7683f74793f627034ee64af8bac8fdd15ebc0e50127cc9779668091ca3410398
+size 2151680456

model-00030-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a713332d8255b32f06f8ca0ff73b6a847c2e788f3086d26e921aa927b1630b2
+size 2264926776

model-00031-of-00413.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ddc6b163c1970bce84c2c78a34e426873dadda5b5ef1d574e044f914ffae47d
+size 2151680448