We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
about some small bug of prompt_tuning.py
### System Info

peft ==0.5.0
python == 3.9
transformers==4.33.1

### Who can help?

_No response_

### Information

- [ ] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder
- [ ] My own task or dataset (give details below)

### Reproduction

```python
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PromptTuningConfig, get_peft_model, TaskType, PromptTuningInit
import torch
tokenizer = AutoTokenizer.from_pretrained("/upp/xgen/xgen-7b-8k-base", trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained("/upp/xgen/xgen-7b-8k-base", torch_dtype=torch.bfloat16,trust_remote_code=True)
config = PromptTuningConfig(task_type=TaskType.CAUSAL_LM,
                            prompt_tuning_init=PromptTuningInit.TEXT,
                            prompt_tuning_init_text="下面是一段人与机器人的对话。",
                            num_virtual_tokens=len(tokenizer("下面是一段人与机器人的对话。")["input_ids"]),
                            tokenizer_name_or_path="xxxxx")  #(local file)

model = get_peft_model(model, config)
```

### Expected behavior

i have an advice of get_peft_model this method , in this function ,have an class PromptEmbedding in  prompt_tuning.py
and line 112 tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path) should have an args trust_remote_code=True 
i met an issue Tokenizer class xxxx  does not exist or is not currently imported. because of it .


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/peft/tuners/prompt_tuning/config.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2023-present the HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 import enum
17 from dataclasses import dataclass, field
18 from typing import Optional, Union
19 
20 from peft.config import PromptLearningConfig
21 from peft.utils import PeftType
22 
23 
24 class PromptTuningInit(str, enum.Enum):
25     TEXT = "TEXT"
26     RANDOM = "RANDOM"
27 
28 
29 @dataclass
30 class PromptTuningConfig(PromptLearningConfig):
31     """
32     This is the configuration class to store the configuration of a [`PromptEmbedding`].
33 
34     Args:
35         prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding.
36         prompt_tuning_init_text (`str`, *optional*):
37             The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.
38         tokenizer_name_or_path (`str`, *optional*):
39             The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.
40     """
41 
42     prompt_tuning_init: Union[PromptTuningInit, str] = field(
43         default=PromptTuningInit.RANDOM,
44         metadata={"help": "How to initialize the prompt tuning parameters"},
45     )
46     prompt_tuning_init_text: Optional[str] = field(
47         default=None,
48         metadata={
49             "help": "The text to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`"
50         },
51     )
52     tokenizer_name_or_path: Optional[str] = field(
53         default=None,
54         metadata={
55             "help": "The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`"
56         },
57     )
58 
59     def __post_init__(self):
60         self.peft_type = PeftType.PROMPT_TUNING
61
```

Path: `src/peft/tuners/prompt_tuning/model.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2023-present the HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 import math
17 
18 import torch
19 
20 from .config import PromptTuningInit
21 
22 
23 class PromptEmbedding(torch.nn.Module):
24     """
25     The model to encode virtual tokens into prompt embeddings.
26 
27     Args:
28         config ([`PromptTuningConfig`]): The configuration of the prompt embedding.
29         word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model.
30 
31     **Attributes**:
32         - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.
33 
34     Example:
35 
36     ```py
37     >>> from peft import PromptEmbedding, PromptTuningConfig
38 
39     >>> config = PromptTuningConfig(
40     ...     peft_type="PROMPT_TUNING",
41     ...     task_type="SEQ_2_SEQ_LM",
42     ...     num_virtual_tokens=20,
43     ...     token_dim=768,
44     ...     num_transformer_submodules=1,
45     ...     num_attention_heads=12,
46     ...     num_layers=12,
47     ...     prompt_tuning_init="TEXT",
48     ...     prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral",
49     ...     tokenizer_name_or_path="t5-base",
50     ... )
51 
52     >>> # t5_model.shared is the word embeddings of the base model
53     >>> prompt_embedding = PromptEmbedding(config, t5_model.shared)
54     ```
55 
56     Input Shape: (`batch_size`, `total_virtual_tokens`)
57 
58     Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)
59     """
60 
61     def __init__(self, config, word_embeddings):
62         super().__init__()
63 
64         total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules
65         self.embedding = torch.nn.Embedding(total_virtual_tokens, config.token_dim)
66         if config.prompt_tuning_init == PromptTuningInit.TEXT:
67             from transformers import AutoTokenizer
68 
69             tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path)
70             init_text = config.prompt_tuning_init_text
71             init_token_ids = tokenizer(init_text)["input_ids"]
72             # Trim or iterate until num_text_tokens matches total_virtual_tokens
73             num_text_tokens = len(init_token_ids)
74             if num_text_tokens > total_virtual_tokens:
75                 init_token_ids = init_token_ids[:total_virtual_tokens]
76             elif num_text_tokens < total_virtual_tokens:
77                 num_reps = math.ceil(total_virtual_tokens / num_text_tokens)
78                 init_token_ids = init_token_ids * num_reps
79             init_token_ids = init_token_ids[:total_virtual_tokens]
80 
81             word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone()
82             word_embedding_weights = word_embedding_weights.to(torch.float32)
83             self.embedding.weight = torch.nn.Parameter(word_embedding_weights)
84 
85     def forward(self, indices):
86         # Just get embeddings
87         prompt_embeddings = self.embedding(indices)
88         return prompt_embeddings
89
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/peft/tuners/prompt_tuning/config.py b/src/peft/tuners/prompt_tuning/config.py
--- a/src/peft/tuners/prompt_tuning/config.py
+++ b/src/peft/tuners/prompt_tuning/config.py
@@ -37,6 +37,9 @@
             The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.
         tokenizer_name_or_path (`str`, *optional*):
             The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.
+        tokenizer_kwargs (`dict`, *optional*):
+            The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if `prompt_tuning_init` is
+            `TEXT`.
     """
 
     prompt_tuning_init: Union[PromptTuningInit, str] = field(
@@ -56,5 +59,18 @@
         },
     )
 
+    tokenizer_kwargs: Optional[dict] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if prompt_tuning_init is "
+                "`TEXT`"
+            ),
+        },
+    )
+
     def __post_init__(self):
         self.peft_type = PeftType.PROMPT_TUNING
+
+        if self.tokenizer_kwargs and (self.prompt_tuning_init != PromptTuningInit.TEXT):
+            raise ValueError(f"tokenizer_kwargs only valid when using prompt_tuning_init='{PromptTuningInit.TEXT}'.")
diff --git a/src/peft/tuners/prompt_tuning/model.py b/src/peft/tuners/prompt_tuning/model.py
--- a/src/peft/tuners/prompt_tuning/model.py
+++ b/src/peft/tuners/prompt_tuning/model.py
@@ -66,7 +66,8 @@
         if config.prompt_tuning_init == PromptTuningInit.TEXT:
             from transformers import AutoTokenizer
 
-            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path)
+            tokenizer_kwargs = config.tokenizer_kwargs or {}
+            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path, **tokenizer_kwargs)
             init_text = config.prompt_tuning_init_text
             init_token_ids = tokenizer(init_text)["input_ids"]
             # Trim or iterate until num_text_tokens matches total_virtual_tokens
@@ -77,8 +78,9 @@
                 num_reps = math.ceil(total_virtual_tokens / num_text_tokens)
                 init_token_ids = init_token_ids * num_reps
             init_token_ids = init_token_ids[:total_virtual_tokens]
+            init_token_ids = torch.LongTensor(init_token_ids).to(word_embeddings.weight.device)
 
-            word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone()
+            word_embedding_weights = word_embeddings(init_token_ids).detach().clone()
             word_embedding_weights = word_embedding_weights.to(torch.float32)
             self.embedding.weight = torch.nn.Parameter(word_embedding_weights)

{"golden_diff": "diff --git a/src/peft/tuners/prompt_tuning/config.py b/src/peft/tuners/prompt_tuning/config.py\n--- a/src/peft/tuners/prompt_tuning/config.py\n+++ b/src/peft/tuners/prompt_tuning/config.py\n@@ -37,6 +37,9 @@\n             The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.\n         tokenizer_name_or_path (`str`, *optional*):\n             The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.\n+        tokenizer_kwargs (`dict`, *optional*):\n+            The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if `prompt_tuning_init` is\n+            `TEXT`.\n     \"\"\"\n \n     prompt_tuning_init: Union[PromptTuningInit, str] = field(\n@@ -56,5 +59,18 @@\n         },\n     )\n \n+    tokenizer_kwargs: Optional[dict] = field(\n+        default=None,\n+        metadata={\n+            \"help\": (\n+                \"The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if prompt_tuning_init is \"\n+                \"`TEXT`\"\n+            ),\n+        },\n+    )\n+\n     def __post_init__(self):\n         self.peft_type = PeftType.PROMPT_TUNING\n+\n+        if self.tokenizer_kwargs and (self.prompt_tuning_init != PromptTuningInit.TEXT):\n+            raise ValueError(f\"tokenizer_kwargs only valid when using prompt_tuning_init='{PromptTuningInit.TEXT}'.\")\ndiff --git a/src/peft/tuners/prompt_tuning/model.py b/src/peft/tuners/prompt_tuning/model.py\n--- a/src/peft/tuners/prompt_tuning/model.py\n+++ b/src/peft/tuners/prompt_tuning/model.py\n@@ -66,7 +66,8 @@\n         if config.prompt_tuning_init == PromptTuningInit.TEXT:\n             from transformers import AutoTokenizer\n \n-            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path)\n+            tokenizer_kwargs = config.tokenizer_kwargs or {}\n+            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path, **tokenizer_kwargs)\n             init_text = config.prompt_tuning_init_text\n             init_token_ids = tokenizer(init_text)[\"input_ids\"]\n             # Trim or iterate until num_text_tokens matches total_virtual_tokens\n@@ -77,8 +78,9 @@\n                 num_reps = math.ceil(total_virtual_tokens / num_text_tokens)\n                 init_token_ids = init_token_ids * num_reps\n             init_token_ids = init_token_ids[:total_virtual_tokens]\n+            init_token_ids = torch.LongTensor(init_token_ids).to(word_embeddings.weight.device)\n \n-            word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone()\n+            word_embedding_weights = word_embeddings(init_token_ids).detach().clone()\n             word_embedding_weights = word_embedding_weights.to(torch.float32)\n             self.embedding.weight = torch.nn.Parameter(word_embedding_weights)\n", "issue": "about some small bug of prompt_tuning.py\n### System Info\r\n\r\npeft ==0.5.0\r\npython == 3.9\r\ntransformers==4.33.1\r\n\r\n### Who can help?\r\n\r\n_No response_\r\n\r\n### Information\r\n\r\n- [ ] The official example scripts\r\n- [ ] My own modified scripts\r\n\r\n### Tasks\r\n\r\n- [ ] An officially supported task in the `examples` folder\r\n- [ ] My own task or dataset (give details below)\r\n\r\n### Reproduction\r\n\r\n```python\r\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\r\nfrom peft import PromptTuningConfig, get_peft_model, TaskType, PromptTuningInit\r\nimport torch\r\ntokenizer = AutoTokenizer.from_pretrained(\"/upp/xgen/xgen-7b-8k-base\", trust_remote_code=True)\r\nmodel = AutoModelForCausalLM.from_pretrained(\"/upp/xgen/xgen-7b-8k-base\", torch_dtype=torch.bfloat16,trust_remote_code=True)\r\nconfig = PromptTuningConfig(task_type=TaskType.CAUSAL_LM,\r\n                            prompt_tuning_init=PromptTuningInit.TEXT,\r\n                            prompt_tuning_init_text=\"\u4e0b\u9762\u662f\u4e00\u6bb5\u4eba\u4e0e\u673a\u5668\u4eba\u7684\u5bf9\u8bdd\u3002\",\r\n                            num_virtual_tokens=len(tokenizer(\"\u4e0b\u9762\u662f\u4e00\u6bb5\u4eba\u4e0e\u673a\u5668\u4eba\u7684\u5bf9\u8bdd\u3002\")[\"input_ids\"]),\r\n                            tokenizer_name_or_path=\"xxxxx\")  #(local file)\r\n\r\nmodel = get_peft_model(model, config)\r\n```\r\n\r\n### Expected behavior\r\n\r\ni have an advice of get_peft_model this method , in this function ,have an class PromptEmbedding in  prompt_tuning.py\r\nand line 112 tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path) should have an args trust_remote_code=True \r\ni met an issue Tokenizer class xxxx  does not exist or is not currently imported. because of it .\r\n\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport enum\nfrom dataclasses import dataclass, field\nfrom typing import Optional, Union\n\nfrom peft.config import PromptLearningConfig\nfrom peft.utils import PeftType\n\n\nclass PromptTuningInit(str, enum.Enum):\n    TEXT = \"TEXT\"\n    RANDOM = \"RANDOM\"\n\n\n@dataclass\nclass PromptTuningConfig(PromptLearningConfig):\n    \"\"\"\n    This is the configuration class to store the configuration of a [`PromptEmbedding`].\n\n    Args:\n        prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding.\n        prompt_tuning_init_text (`str`, *optional*):\n            The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.\n        tokenizer_name_or_path (`str`, *optional*):\n            The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.\n    \"\"\"\n\n    prompt_tuning_init: Union[PromptTuningInit, str] = field(\n        default=PromptTuningInit.RANDOM,\n        metadata={\"help\": \"How to initialize the prompt tuning parameters\"},\n    )\n    prompt_tuning_init_text: Optional[str] = field(\n        default=None,\n        metadata={\n            \"help\": \"The text to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`\"\n        },\n    )\n    tokenizer_name_or_path: Optional[str] = field(\n        default=None,\n        metadata={\n            \"help\": \"The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`\"\n        },\n    )\n\n    def __post_init__(self):\n        self.peft_type = PeftType.PROMPT_TUNING\n", "path": "src/peft/tuners/prompt_tuning/config.py"}, {"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport math\n\nimport torch\n\nfrom .config import PromptTuningInit\n\n\nclass PromptEmbedding(torch.nn.Module):\n    \"\"\"\n    The model to encode virtual tokens into prompt embeddings.\n\n    Args:\n        config ([`PromptTuningConfig`]): The configuration of the prompt embedding.\n        word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model.\n\n    **Attributes**:\n        - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.\n\n    Example:\n\n    ```py\n    >>> from peft import PromptEmbedding, PromptTuningConfig\n\n    >>> config = PromptTuningConfig(\n    ...     peft_type=\"PROMPT_TUNING\",\n    ...     task_type=\"SEQ_2_SEQ_LM\",\n    ...     num_virtual_tokens=20,\n    ...     token_dim=768,\n    ...     num_transformer_submodules=1,\n    ...     num_attention_heads=12,\n    ...     num_layers=12,\n    ...     prompt_tuning_init=\"TEXT\",\n    ...     prompt_tuning_init_text=\"Predict if sentiment of this review is positive, negative or neutral\",\n    ...     tokenizer_name_or_path=\"t5-base\",\n    ... )\n\n    >>> # t5_model.shared is the word embeddings of the base model\n    >>> prompt_embedding = PromptEmbedding(config, t5_model.shared)\n    ```\n\n    Input Shape: (`batch_size`, `total_virtual_tokens`)\n\n    Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)\n    \"\"\"\n\n    def __init__(self, config, word_embeddings):\n        super().__init__()\n\n        total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules\n        self.embedding = torch.nn.Embedding(total_virtual_tokens, config.token_dim)\n        if config.prompt_tuning_init == PromptTuningInit.TEXT:\n            from transformers import AutoTokenizer\n\n            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path)\n            init_text = config.prompt_tuning_init_text\n            init_token_ids = tokenizer(init_text)[\"input_ids\"]\n            # Trim or iterate until num_text_tokens matches total_virtual_tokens\n            num_text_tokens = len(init_token_ids)\n            if num_text_tokens > total_virtual_tokens:\n                init_token_ids = init_token_ids[:total_virtual_tokens]\n            elif num_text_tokens < total_virtual_tokens:\n                num_reps = math.ceil(total_virtual_tokens / num_text_tokens)\n                init_token_ids = init_token_ids * num_reps\n            init_token_ids = init_token_ids[:total_virtual_tokens]\n\n            word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone()\n            word_embedding_weights = word_embedding_weights.to(torch.float32)\n            self.embedding.weight = torch.nn.Parameter(word_embedding_weights)\n\n    def forward(self, indices):\n        # Just get embeddings\n        prompt_embeddings = self.embedding(indices)\n        return prompt_embeddings\n", "path": "src/peft/tuners/prompt_tuning/model.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport enum\nfrom dataclasses import dataclass, field\nfrom typing import Optional, Union\n\nfrom peft.config import PromptLearningConfig\nfrom peft.utils import PeftType\n\n\nclass PromptTuningInit(str, enum.Enum):\n    TEXT = \"TEXT\"\n    RANDOM = \"RANDOM\"\n\n\n@dataclass\nclass PromptTuningConfig(PromptLearningConfig):\n    \"\"\"\n    This is the configuration class to store the configuration of a [`PromptEmbedding`].\n\n    Args:\n        prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding.\n        prompt_tuning_init_text (`str`, *optional*):\n            The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.\n        tokenizer_name_or_path (`str`, *optional*):\n            The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.\n        tokenizer_kwargs (`dict`, *optional*):\n            The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if `prompt_tuning_init` is\n            `TEXT`.\n    \"\"\"\n\n    prompt_tuning_init: Union[PromptTuningInit, str] = field(\n        default=PromptTuningInit.RANDOM,\n        metadata={\"help\": \"How to initialize the prompt tuning parameters\"},\n    )\n    prompt_tuning_init_text: Optional[str] = field(\n        default=None,\n        metadata={\n            \"help\": \"The text to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`\"\n        },\n    )\n    tokenizer_name_or_path: Optional[str] = field(\n        default=None,\n        metadata={\n            \"help\": \"The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`\"\n        },\n    )\n\n    tokenizer_kwargs: Optional[dict] = field(\n        default=None,\n        metadata={\n            \"help\": (\n                \"The keyword arguments to pass to `AutoTokenizer.from_pretrained`. Only used if prompt_tuning_init is \"\n                \"`TEXT`\"\n            ),\n        },\n    )\n\n    def __post_init__(self):\n        self.peft_type = PeftType.PROMPT_TUNING\n\n        if self.tokenizer_kwargs and (self.prompt_tuning_init != PromptTuningInit.TEXT):\n            raise ValueError(f\"tokenizer_kwargs only valid when using prompt_tuning_init='{PromptTuningInit.TEXT}'.\")\n", "path": "src/peft/tuners/prompt_tuning/config.py"}, {"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport math\n\nimport torch\n\nfrom .config import PromptTuningInit\n\n\nclass PromptEmbedding(torch.nn.Module):\n    \"\"\"\n    The model to encode virtual tokens into prompt embeddings.\n\n    Args:\n        config ([`PromptTuningConfig`]): The configuration of the prompt embedding.\n        word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model.\n\n    **Attributes**:\n        - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.\n\n    Example:\n\n    ```py\n    >>> from peft import PromptEmbedding, PromptTuningConfig\n\n    >>> config = PromptTuningConfig(\n    ...     peft_type=\"PROMPT_TUNING\",\n    ...     task_type=\"SEQ_2_SEQ_LM\",\n    ...     num_virtual_tokens=20,\n    ...     token_dim=768,\n    ...     num_transformer_submodules=1,\n    ...     num_attention_heads=12,\n    ...     num_layers=12,\n    ...     prompt_tuning_init=\"TEXT\",\n    ...     prompt_tuning_init_text=\"Predict if sentiment of this review is positive, negative or neutral\",\n    ...     tokenizer_name_or_path=\"t5-base\",\n    ... )\n\n    >>> # t5_model.shared is the word embeddings of the base model\n    >>> prompt_embedding = PromptEmbedding(config, t5_model.shared)\n    ```\n\n    Input Shape: (`batch_size`, `total_virtual_tokens`)\n\n    Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)\n    \"\"\"\n\n    def __init__(self, config, word_embeddings):\n        super().__init__()\n\n        total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules\n        self.embedding = torch.nn.Embedding(total_virtual_tokens, config.token_dim)\n        if config.prompt_tuning_init == PromptTuningInit.TEXT:\n            from transformers import AutoTokenizer\n\n            tokenizer_kwargs = config.tokenizer_kwargs or {}\n            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path, **tokenizer_kwargs)\n            init_text = config.prompt_tuning_init_text\n            init_token_ids = tokenizer(init_text)[\"input_ids\"]\n            # Trim or iterate until num_text_tokens matches total_virtual_tokens\n            num_text_tokens = len(init_token_ids)\n            if num_text_tokens > total_virtual_tokens:\n                init_token_ids = init_token_ids[:total_virtual_tokens]\n            elif num_text_tokens < total_virtual_tokens:\n                num_reps = math.ceil(total_virtual_tokens / num_text_tokens)\n                init_token_ids = init_token_ids * num_reps\n            init_token_ids = init_token_ids[:total_virtual_tokens]\n            init_token_ids = torch.LongTensor(init_token_ids).to(word_embeddings.weight.device)\n\n            word_embedding_weights = word_embeddings(init_token_ids).detach().clone()\n            word_embedding_weights = word_embedding_weights.to(torch.float32)\n            self.embedding.weight = torch.nn.Parameter(word_embedding_weights)\n\n    def forward(self, indices):\n        # Just get embeddings\n        prompt_embeddings = self.embedding(indices)\n        return prompt_embeddings\n", "path": "src/peft/tuners/prompt_tuning/model.py"}]}