We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
An error occurs when using LoftQ. IndexError
A problem occurred while applying LoftQ to T5.
`IndexError: tensors used as indices must be long, byte or bool tensors`
The problem was in [this line](https://github.com/huggingface/peft/blob/main/src/peft/utils/loftq_utils.py#L158), I replaced int with long and it worked fine.
`lookup_table_idx = lookup_table_idx.to(torch.long)`

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/peft/utils/loftq_utils.py`
Content:
```
1 # coding=utf-8
2 # Copyright 2023-present the HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 # Reference code: https://github.com/yxli2123/LoftQ/blob/main/utils.py
17 # Reference paper: https://arxiv.org/abs/2310.08659
18 
19 import logging
20 from typing import Union
21 
22 import torch
23 
24 from peft.import_utils import is_bnb_4bit_available, is_bnb_available
25 
26 
27 if is_bnb_available():
28     import bitsandbytes as bnb
29 
30 
31 class NFQuantizer:
32     def __init__(self, num_bits=2, device="cuda", method="normal", block_size=64, *args, **kwargs):
33         super().__init__(*args, **kwargs)
34         self.num_bits = num_bits
35         self.device = device
36         self.method = method
37         self.block_size = block_size
38         if self.method == "normal":
39             self.norm_lookup_table = self.create_normal_map(num_bits=self.num_bits)
40             self.norm_lookup_table = self.norm_lookup_table.to(device)
41         elif self.method == "uniform":
42             self.norm_lookup_table = self.create_uniform_map(num_bits=self.num_bits)
43             self.norm_lookup_table = self.norm_lookup_table.to(device)
44         else:
45             raise NotImplementedError("Other quantization methods not supported yet.")
46 
47     @staticmethod
48     def create_uniform_map(symmetric=False, num_bits=4):
49         if symmetric:
50             # print("symmetric uniform quantization")
51             negative = torch.linspace(-1, 0, 2 ** (num_bits - 1))
52             positive = torch.linspace(0, 1, 2 ** (num_bits - 1))
53             table = torch.cat([negative, positive[1:]])
54         else:
55             # print("asymmetric uniform quantization")
56             table = torch.linspace(-1, 1, 2**num_bits)
57         return table
58 
59     @staticmethod
60     def create_normal_map(offset=0.9677083, symmetric=False, num_bits=2):
61         try:
62             from scipy.stats import norm
63         except ImportError:
64             raise ImportError("The required package 'scipy' is not installed. Please install it to continue.")
65 
66         variations = 2**num_bits
67         if symmetric:
68             v = norm.ppf(torch.linspace(1 - offset, offset, variations + 1)).tolist()
69             values = []
70             for index in range(len(v) - 1):
71                 values.append(0.5 * v[index] + 0.5 * v[index + 1])
72             v = values
73         else:
74             # one more positive value, this is an asymmetric type
75             v1 = norm.ppf(torch.linspace(offset, 0.5, variations // 2 + 1)[:-1]).tolist()
76             v2 = [0]
77             v3 = (-norm.ppf(torch.linspace(offset, 0.5, variations // 2)[:-1])).tolist()
78             v = v1 + v2 + v3
79 
80         values = torch.Tensor(v)
81         values = values.sort().values
82         values /= values.max()
83         return values
84 
85     def quantize_tensor(self, weight):
86         max_abs = torch.abs(weight).max()
87         weight_normed = weight / max_abs
88 
89         weight_normed_expanded = weight_normed.unsqueeze(-1)
90 
91         # Reshape L to have the same number of dimensions as X_expanded
92         L_reshaped = torch.tensor(self.norm_lookup_table).reshape(1, -1)
93 
94         # Calculate the absolute difference between X_expanded and L_reshaped
95         abs_diff = torch.abs(weight_normed_expanded - L_reshaped)
96 
97         # Find the index of the minimum absolute difference for each element
98         qweight = torch.argmin(abs_diff, dim=-1)
99         return qweight, max_abs
100 
101     def dequantize_tensor(self, qweight, max_abs):
102         qweight_flatten = qweight.flatten()
103 
104         weight_normed = self.norm_lookup_table[qweight_flatten]
105         weight = weight_normed * max_abs
106 
107         weight = weight.reshape(qweight.shape)
108 
109         return weight
110 
111     def quantize_block(self, weight):
112         if len(weight.shape) != 2:
113             raise ValueError(f"Only support 2D matrix, but your input has {len(weight.shape)} dimensions.")
114         if weight.shape[0] * weight.shape[1] % self.block_size != 0:
115             raise ValueError(
116                 f"Weight with shape ({weight.shape[0]} x {weight.shape[1]}) "
117                 f"is not dividable by block size {self.block_size}."
118             )
119 
120         M, N = weight.shape
121         device = weight.device
122 
123         # Quantization
124         weight_flatten = weight.flatten()  # (M*N, )
125         weight_block = weight_flatten.reshape(-1, self.block_size)  # (L, B), L = M * N / B
126         if self.method == "normal":
127             weight_max = weight_block.abs().max(dim=-1)[0]  # (L, 1)
128         elif self.method == "uniform":
129             weight_max = weight_block.mean(dim=-1) + 2.5 * weight_block.std(dim=-1)
130         else:
131             raise NotImplementedError("Method not supported yet.")
132         weight_max = weight_max.unsqueeze(-1)
133         weight_divabs = weight_block / weight_max  # (L, B)
134         weight_divabs = weight_divabs.unsqueeze(-1)  # (L, B, 1)
135         L_reshaped = self.norm_lookup_table.reshape(1, -1)  # (1, 2**K)
136 
137         abs_diff = torch.abs(weight_divabs - L_reshaped)  # (L, B, 2**K)
138         qweight = torch.argmin(abs_diff, dim=-1)  # (L, B)
139 
140         # Pack multiple k-bit into uint8
141         qweight = qweight.reshape(-1, 8 // self.num_bits)
142         qweight_pack = torch.zeros((M * N // 8 * self.num_bits, 1), dtype=torch.uint8, device=device)
143 
144         # data format example:
145         # [1, 0, 3, 2] or [01, 00, 11, 10]  -> [10110001], LIFO
146         for i in range(8 // self.num_bits):
147             qweight[:, i] = qweight[:, i] << i * self.num_bits
148             qweight_pack[:, 0] |= qweight[:, i]
149 
150         return qweight_pack, weight_max, weight.shape
151 
152     def dequantize_block(self, qweight, weight_max, weight_shape):
153         # unpack weight
154         device = qweight.device
155         weight = torch.zeros((qweight.shape[0], 8 // self.num_bits), dtype=torch.float32, device=device)
156         for i in range(8 // self.num_bits):
157             lookup_table_idx = qweight.to(torch.long) % 2**self.num_bits  # get the most right 2 bits
158             lookup_table_idx = lookup_table_idx.to(torch.int)
159             weight[:, i] = self.norm_lookup_table[lookup_table_idx].squeeze()
160             qweight = qweight >> self.num_bits  # right shift 2 bits of the original data
161 
162         weight_block = weight.reshape(-1, self.block_size)
163         weight = weight_block * weight_max
164         weight = weight.reshape(weight_shape)
165 
166         return weight
167 
168 
169 def _low_rank_decomposition(weight, reduced_rank=32):
170     """
171     :param weight: The matrix to decompose, of shape (H, W) :param reduced_rank: the final rank :return:
172     """
173     matrix_dimension = len(weight.size())
174     if matrix_dimension != 2:
175         raise ValueError(f"Only support 2D matrix, but your input has {matrix_dimension} dimensions.")
176 
177     # Use SVD to decompose a matrix, default full_matrices is False to save parameters
178     U, S, Vh = torch.linalg.svd(weight, full_matrices=False)
179 
180     L = U @ (torch.sqrt(torch.diag(S)[:, 0:reduced_rank]))
181     R = torch.sqrt(torch.diag(S)[0:reduced_rank, :]) @ Vh
182 
183     return {"L": L, "R": R, "U": U, "S": S, "Vh": Vh, "reduced_rank": reduced_rank}
184 
185 
186 @torch.no_grad()
187 def loftq_init(weight: Union[torch.Tensor, torch.nn.Parameter], num_bits: int, reduced_rank: int, num_iter=1):
188     if num_bits not in [2, 4, 8]:
189         raise ValueError("Only support 2, 4, 8 bits quantization")
190     if num_iter <= 0:
191         raise ValueError("Number of iterations must be greater than 0")
192 
193     out_feature, in_feature = weight.size()
194     device = weight.device
195     dtype = weight.dtype
196 
197     logging.info(
198         f"Weight: ({out_feature}, {in_feature}) | Rank: {reduced_rank} "
199         f"| Num Iter: {num_iter} | Num Bits: {num_bits}"
200     )
201     if not is_bnb_4bit_available() or num_bits in [2, 8]:
202         quantizer = NFQuantizer(num_bits=num_bits, device=device, method="normal", block_size=64)
203         compute_device = device
204     else:
205         compute_device = "cuda"
206 
207     weight = weight.to(device=compute_device, dtype=torch.float32)
208     res = weight.clone()
209     for i in range(num_iter):
210         torch.cuda.empty_cache()
211         # Quantization
212         if num_bits == 4 and is_bnb_4bit_available():
213             qweight = bnb.nn.Params4bit(
214                 res.to("cpu"), requires_grad=False, compress_statistics=False, quant_type="nf4"
215             ).to(compute_device)
216             dequantized_weight = bnb.functional.dequantize_4bit(qweight.data, qweight.quant_state)
217         else:
218             quantized_weight, max_abs, shape = quantizer.quantize_block(res)
219             dequantized_weight = quantizer.dequantize_block(quantized_weight, max_abs, shape)
220 
221         res = weight - dequantized_weight
222 
223         # Decompose the residual by SVD
224         output = _low_rank_decomposition(res, reduced_rank=reduced_rank)
225         L, R, reduced_rank = output["L"], output["R"], output["reduced_rank"]
226         res = weight - torch.mm(L, R)
227 
228     lora_A, lora_B = R, L
229 
230     return dequantized_weight.to(device=device, dtype=dtype), lora_A, lora_B
231
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/peft/utils/loftq_utils.py b/src/peft/utils/loftq_utils.py
--- a/src/peft/utils/loftq_utils.py
+++ b/src/peft/utils/loftq_utils.py
@@ -155,7 +155,7 @@
         weight = torch.zeros((qweight.shape[0], 8 // self.num_bits), dtype=torch.float32, device=device)
         for i in range(8 // self.num_bits):
             lookup_table_idx = qweight.to(torch.long) % 2**self.num_bits  # get the most right 2 bits
-            lookup_table_idx = lookup_table_idx.to(torch.int)
+            lookup_table_idx = lookup_table_idx.to(torch.long)
             weight[:, i] = self.norm_lookup_table[lookup_table_idx].squeeze()
             qweight = qweight >> self.num_bits  # right shift 2 bits of the original data

{"golden_diff": "diff --git a/src/peft/utils/loftq_utils.py b/src/peft/utils/loftq_utils.py\n--- a/src/peft/utils/loftq_utils.py\n+++ b/src/peft/utils/loftq_utils.py\n@@ -155,7 +155,7 @@\n         weight = torch.zeros((qweight.shape[0], 8 // self.num_bits), dtype=torch.float32, device=device)\n         for i in range(8 // self.num_bits):\n             lookup_table_idx = qweight.to(torch.long) % 2**self.num_bits  # get the most right 2 bits\n-            lookup_table_idx = lookup_table_idx.to(torch.int)\n+            lookup_table_idx = lookup_table_idx.to(torch.long)\n             weight[:, i] = self.norm_lookup_table[lookup_table_idx].squeeze()\n             qweight = qweight >> self.num_bits  # right shift 2 bits of the original data\n", "issue": "An error occurs when using LoftQ. IndexError\nA problem occurred while applying LoftQ to T5.\r\n`IndexError: tensors used as indices must be long, byte or bool tensors`\r\nThe problem was in [this line](https://github.com/huggingface/peft/blob/main/src/peft/utils/loftq_utils.py#L158), I replaced int with long and it worked fine.\r\n`lookup_table_idx = lookup_table_idx.to(torch.long)`\n", "before_files": [{"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Reference code: https://github.com/yxli2123/LoftQ/blob/main/utils.py\n# Reference paper: https://arxiv.org/abs/2310.08659\n\nimport logging\nfrom typing import Union\n\nimport torch\n\nfrom peft.import_utils import is_bnb_4bit_available, is_bnb_available\n\n\nif is_bnb_available():\n    import bitsandbytes as bnb\n\n\nclass NFQuantizer:\n    def __init__(self, num_bits=2, device=\"cuda\", method=\"normal\", block_size=64, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.num_bits = num_bits\n        self.device = device\n        self.method = method\n        self.block_size = block_size\n        if self.method == \"normal\":\n            self.norm_lookup_table = self.create_normal_map(num_bits=self.num_bits)\n            self.norm_lookup_table = self.norm_lookup_table.to(device)\n        elif self.method == \"uniform\":\n            self.norm_lookup_table = self.create_uniform_map(num_bits=self.num_bits)\n            self.norm_lookup_table = self.norm_lookup_table.to(device)\n        else:\n            raise NotImplementedError(\"Other quantization methods not supported yet.\")\n\n    @staticmethod\n    def create_uniform_map(symmetric=False, num_bits=4):\n        if symmetric:\n            # print(\"symmetric uniform quantization\")\n            negative = torch.linspace(-1, 0, 2 ** (num_bits - 1))\n            positive = torch.linspace(0, 1, 2 ** (num_bits - 1))\n            table = torch.cat([negative, positive[1:]])\n        else:\n            # print(\"asymmetric uniform quantization\")\n            table = torch.linspace(-1, 1, 2**num_bits)\n        return table\n\n    @staticmethod\n    def create_normal_map(offset=0.9677083, symmetric=False, num_bits=2):\n        try:\n            from scipy.stats import norm\n        except ImportError:\n            raise ImportError(\"The required package 'scipy' is not installed. Please install it to continue.\")\n\n        variations = 2**num_bits\n        if symmetric:\n            v = norm.ppf(torch.linspace(1 - offset, offset, variations + 1)).tolist()\n            values = []\n            for index in range(len(v) - 1):\n                values.append(0.5 * v[index] + 0.5 * v[index + 1])\n            v = values\n        else:\n            # one more positive value, this is an asymmetric type\n            v1 = norm.ppf(torch.linspace(offset, 0.5, variations // 2 + 1)[:-1]).tolist()\n            v2 = [0]\n            v3 = (-norm.ppf(torch.linspace(offset, 0.5, variations // 2)[:-1])).tolist()\n            v = v1 + v2 + v3\n\n        values = torch.Tensor(v)\n        values = values.sort().values\n        values /= values.max()\n        return values\n\n    def quantize_tensor(self, weight):\n        max_abs = torch.abs(weight).max()\n        weight_normed = weight / max_abs\n\n        weight_normed_expanded = weight_normed.unsqueeze(-1)\n\n        # Reshape L to have the same number of dimensions as X_expanded\n        L_reshaped = torch.tensor(self.norm_lookup_table).reshape(1, -1)\n\n        # Calculate the absolute difference between X_expanded and L_reshaped\n        abs_diff = torch.abs(weight_normed_expanded - L_reshaped)\n\n        # Find the index of the minimum absolute difference for each element\n        qweight = torch.argmin(abs_diff, dim=-1)\n        return qweight, max_abs\n\n    def dequantize_tensor(self, qweight, max_abs):\n        qweight_flatten = qweight.flatten()\n\n        weight_normed = self.norm_lookup_table[qweight_flatten]\n        weight = weight_normed * max_abs\n\n        weight = weight.reshape(qweight.shape)\n\n        return weight\n\n    def quantize_block(self, weight):\n        if len(weight.shape) != 2:\n            raise ValueError(f\"Only support 2D matrix, but your input has {len(weight.shape)} dimensions.\")\n        if weight.shape[0] * weight.shape[1] % self.block_size != 0:\n            raise ValueError(\n                f\"Weight with shape ({weight.shape[0]} x {weight.shape[1]}) \"\n                f\"is not dividable by block size {self.block_size}.\"\n            )\n\n        M, N = weight.shape\n        device = weight.device\n\n        # Quantization\n        weight_flatten = weight.flatten()  # (M*N, )\n        weight_block = weight_flatten.reshape(-1, self.block_size)  # (L, B), L = M * N / B\n        if self.method == \"normal\":\n            weight_max = weight_block.abs().max(dim=-1)[0]  # (L, 1)\n        elif self.method == \"uniform\":\n            weight_max = weight_block.mean(dim=-1) + 2.5 * weight_block.std(dim=-1)\n        else:\n            raise NotImplementedError(\"Method not supported yet.\")\n        weight_max = weight_max.unsqueeze(-1)\n        weight_divabs = weight_block / weight_max  # (L, B)\n        weight_divabs = weight_divabs.unsqueeze(-1)  # (L, B, 1)\n        L_reshaped = self.norm_lookup_table.reshape(1, -1)  # (1, 2**K)\n\n        abs_diff = torch.abs(weight_divabs - L_reshaped)  # (L, B, 2**K)\n        qweight = torch.argmin(abs_diff, dim=-1)  # (L, B)\n\n        # Pack multiple k-bit into uint8\n        qweight = qweight.reshape(-1, 8 // self.num_bits)\n        qweight_pack = torch.zeros((M * N // 8 * self.num_bits, 1), dtype=torch.uint8, device=device)\n\n        # data format example:\n        # [1, 0, 3, 2] or [01, 00, 11, 10]  -> [10110001], LIFO\n        for i in range(8 // self.num_bits):\n            qweight[:, i] = qweight[:, i] << i * self.num_bits\n            qweight_pack[:, 0] |= qweight[:, i]\n\n        return qweight_pack, weight_max, weight.shape\n\n    def dequantize_block(self, qweight, weight_max, weight_shape):\n        # unpack weight\n        device = qweight.device\n        weight = torch.zeros((qweight.shape[0], 8 // self.num_bits), dtype=torch.float32, device=device)\n        for i in range(8 // self.num_bits):\n            lookup_table_idx = qweight.to(torch.long) % 2**self.num_bits  # get the most right 2 bits\n            lookup_table_idx = lookup_table_idx.to(torch.int)\n            weight[:, i] = self.norm_lookup_table[lookup_table_idx].squeeze()\n            qweight = qweight >> self.num_bits  # right shift 2 bits of the original data\n\n        weight_block = weight.reshape(-1, self.block_size)\n        weight = weight_block * weight_max\n        weight = weight.reshape(weight_shape)\n\n        return weight\n\n\ndef _low_rank_decomposition(weight, reduced_rank=32):\n    \"\"\"\n    :param weight: The matrix to decompose, of shape (H, W) :param reduced_rank: the final rank :return:\n    \"\"\"\n    matrix_dimension = len(weight.size())\n    if matrix_dimension != 2:\n        raise ValueError(f\"Only support 2D matrix, but your input has {matrix_dimension} dimensions.\")\n\n    # Use SVD to decompose a matrix, default full_matrices is False to save parameters\n    U, S, Vh = torch.linalg.svd(weight, full_matrices=False)\n\n    L = U @ (torch.sqrt(torch.diag(S)[:, 0:reduced_rank]))\n    R = torch.sqrt(torch.diag(S)[0:reduced_rank, :]) @ Vh\n\n    return {\"L\": L, \"R\": R, \"U\": U, \"S\": S, \"Vh\": Vh, \"reduced_rank\": reduced_rank}\n\n\n@torch.no_grad()\ndef loftq_init(weight: Union[torch.Tensor, torch.nn.Parameter], num_bits: int, reduced_rank: int, num_iter=1):\n    if num_bits not in [2, 4, 8]:\n        raise ValueError(\"Only support 2, 4, 8 bits quantization\")\n    if num_iter <= 0:\n        raise ValueError(\"Number of iterations must be greater than 0\")\n\n    out_feature, in_feature = weight.size()\n    device = weight.device\n    dtype = weight.dtype\n\n    logging.info(\n        f\"Weight: ({out_feature}, {in_feature}) | Rank: {reduced_rank} \"\n        f\"| Num Iter: {num_iter} | Num Bits: {num_bits}\"\n    )\n    if not is_bnb_4bit_available() or num_bits in [2, 8]:\n        quantizer = NFQuantizer(num_bits=num_bits, device=device, method=\"normal\", block_size=64)\n        compute_device = device\n    else:\n        compute_device = \"cuda\"\n\n    weight = weight.to(device=compute_device, dtype=torch.float32)\n    res = weight.clone()\n    for i in range(num_iter):\n        torch.cuda.empty_cache()\n        # Quantization\n        if num_bits == 4 and is_bnb_4bit_available():\n            qweight = bnb.nn.Params4bit(\n                res.to(\"cpu\"), requires_grad=False, compress_statistics=False, quant_type=\"nf4\"\n            ).to(compute_device)\n            dequantized_weight = bnb.functional.dequantize_4bit(qweight.data, qweight.quant_state)\n        else:\n            quantized_weight, max_abs, shape = quantizer.quantize_block(res)\n            dequantized_weight = quantizer.dequantize_block(quantized_weight, max_abs, shape)\n\n        res = weight - dequantized_weight\n\n        # Decompose the residual by SVD\n        output = _low_rank_decomposition(res, reduced_rank=reduced_rank)\n        L, R, reduced_rank = output[\"L\"], output[\"R\"], output[\"reduced_rank\"]\n        res = weight - torch.mm(L, R)\n\n    lora_A, lora_B = R, L\n\n    return dequantized_weight.to(device=device, dtype=dtype), lora_A, lora_B\n", "path": "src/peft/utils/loftq_utils.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Reference code: https://github.com/yxli2123/LoftQ/blob/main/utils.py\n# Reference paper: https://arxiv.org/abs/2310.08659\n\nimport logging\nfrom typing import Union\n\nimport torch\n\nfrom peft.import_utils import is_bnb_4bit_available, is_bnb_available\n\n\nif is_bnb_available():\n    import bitsandbytes as bnb\n\n\nclass NFQuantizer:\n    def __init__(self, num_bits=2, device=\"cuda\", method=\"normal\", block_size=64, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.num_bits = num_bits\n        self.device = device\n        self.method = method\n        self.block_size = block_size\n        if self.method == \"normal\":\n            self.norm_lookup_table = self.create_normal_map(num_bits=self.num_bits)\n            self.norm_lookup_table = self.norm_lookup_table.to(device)\n        elif self.method == \"uniform\":\n            self.norm_lookup_table = self.create_uniform_map(num_bits=self.num_bits)\n            self.norm_lookup_table = self.norm_lookup_table.to(device)\n        else:\n            raise NotImplementedError(\"Other quantization methods not supported yet.\")\n\n    @staticmethod\n    def create_uniform_map(symmetric=False, num_bits=4):\n        if symmetric:\n            # print(\"symmetric uniform quantization\")\n            negative = torch.linspace(-1, 0, 2 ** (num_bits - 1))\n            positive = torch.linspace(0, 1, 2 ** (num_bits - 1))\n            table = torch.cat([negative, positive[1:]])\n        else:\n            # print(\"asymmetric uniform quantization\")\n            table = torch.linspace(-1, 1, 2**num_bits)\n        return table\n\n    @staticmethod\n    def create_normal_map(offset=0.9677083, symmetric=False, num_bits=2):\n        try:\n            from scipy.stats import norm\n        except ImportError:\n            raise ImportError(\"The required package 'scipy' is not installed. Please install it to continue.\")\n\n        variations = 2**num_bits\n        if symmetric:\n            v = norm.ppf(torch.linspace(1 - offset, offset, variations + 1)).tolist()\n            values = []\n            for index in range(len(v) - 1):\n                values.append(0.5 * v[index] + 0.5 * v[index + 1])\n            v = values\n        else:\n            # one more positive value, this is an asymmetric type\n            v1 = norm.ppf(torch.linspace(offset, 0.5, variations // 2 + 1)[:-1]).tolist()\n            v2 = [0]\n            v3 = (-norm.ppf(torch.linspace(offset, 0.5, variations // 2)[:-1])).tolist()\n            v = v1 + v2 + v3\n\n        values = torch.Tensor(v)\n        values = values.sort().values\n        values /= values.max()\n        return values\n\n    def quantize_tensor(self, weight):\n        max_abs = torch.abs(weight).max()\n        weight_normed = weight / max_abs\n\n        weight_normed_expanded = weight_normed.unsqueeze(-1)\n\n        # Reshape L to have the same number of dimensions as X_expanded\n        L_reshaped = torch.tensor(self.norm_lookup_table).reshape(1, -1)\n\n        # Calculate the absolute difference between X_expanded and L_reshaped\n        abs_diff = torch.abs(weight_normed_expanded - L_reshaped)\n\n        # Find the index of the minimum absolute difference for each element\n        qweight = torch.argmin(abs_diff, dim=-1)\n        return qweight, max_abs\n\n    def dequantize_tensor(self, qweight, max_abs):\n        qweight_flatten = qweight.flatten()\n\n        weight_normed = self.norm_lookup_table[qweight_flatten]\n        weight = weight_normed * max_abs\n\n        weight = weight.reshape(qweight.shape)\n\n        return weight\n\n    def quantize_block(self, weight):\n        if len(weight.shape) != 2:\n            raise ValueError(f\"Only support 2D matrix, but your input has {len(weight.shape)} dimensions.\")\n        if weight.shape[0] * weight.shape[1] % self.block_size != 0:\n            raise ValueError(\n                f\"Weight with shape ({weight.shape[0]} x {weight.shape[1]}) \"\n                f\"is not dividable by block size {self.block_size}.\"\n            )\n\n        M, N = weight.shape\n        device = weight.device\n\n        # Quantization\n        weight_flatten = weight.flatten()  # (M*N, )\n        weight_block = weight_flatten.reshape(-1, self.block_size)  # (L, B), L = M * N / B\n        if self.method == \"normal\":\n            weight_max = weight_block.abs().max(dim=-1)[0]  # (L, 1)\n        elif self.method == \"uniform\":\n            weight_max = weight_block.mean(dim=-1) + 2.5 * weight_block.std(dim=-1)\n        else:\n            raise NotImplementedError(\"Method not supported yet.\")\n        weight_max = weight_max.unsqueeze(-1)\n        weight_divabs = weight_block / weight_max  # (L, B)\n        weight_divabs = weight_divabs.unsqueeze(-1)  # (L, B, 1)\n        L_reshaped = self.norm_lookup_table.reshape(1, -1)  # (1, 2**K)\n\n        abs_diff = torch.abs(weight_divabs - L_reshaped)  # (L, B, 2**K)\n        qweight = torch.argmin(abs_diff, dim=-1)  # (L, B)\n\n        # Pack multiple k-bit into uint8\n        qweight = qweight.reshape(-1, 8 // self.num_bits)\n        qweight_pack = torch.zeros((M * N // 8 * self.num_bits, 1), dtype=torch.uint8, device=device)\n\n        # data format example:\n        # [1, 0, 3, 2] or [01, 00, 11, 10]  -> [10110001], LIFO\n        for i in range(8 // self.num_bits):\n            qweight[:, i] = qweight[:, i] << i * self.num_bits\n            qweight_pack[:, 0] |= qweight[:, i]\n\n        return qweight_pack, weight_max, weight.shape\n\n    def dequantize_block(self, qweight, weight_max, weight_shape):\n        # unpack weight\n        device = qweight.device\n        weight = torch.zeros((qweight.shape[0], 8 // self.num_bits), dtype=torch.float32, device=device)\n        for i in range(8 // self.num_bits):\n            lookup_table_idx = qweight.to(torch.long) % 2**self.num_bits  # get the most right 2 bits\n            lookup_table_idx = lookup_table_idx.to(torch.long)\n            weight[:, i] = self.norm_lookup_table[lookup_table_idx].squeeze()\n            qweight = qweight >> self.num_bits  # right shift 2 bits of the original data\n\n        weight_block = weight.reshape(-1, self.block_size)\n        weight = weight_block * weight_max\n        weight = weight.reshape(weight_shape)\n\n        return weight\n\n\ndef _low_rank_decomposition(weight, reduced_rank=32):\n    \"\"\"\n    :param weight: The matrix to decompose, of shape (H, W) :param reduced_rank: the final rank :return:\n    \"\"\"\n    matrix_dimension = len(weight.size())\n    if matrix_dimension != 2:\n        raise ValueError(f\"Only support 2D matrix, but your input has {matrix_dimension} dimensions.\")\n\n    # Use SVD to decompose a matrix, default full_matrices is False to save parameters\n    U, S, Vh = torch.linalg.svd(weight, full_matrices=False)\n\n    L = U @ (torch.sqrt(torch.diag(S)[:, 0:reduced_rank]))\n    R = torch.sqrt(torch.diag(S)[0:reduced_rank, :]) @ Vh\n\n    return {\"L\": L, \"R\": R, \"U\": U, \"S\": S, \"Vh\": Vh, \"reduced_rank\": reduced_rank}\n\n\n@torch.no_grad()\ndef loftq_init(weight: Union[torch.Tensor, torch.nn.Parameter], num_bits: int, reduced_rank: int, num_iter=1):\n    if num_bits not in [2, 4, 8]:\n        raise ValueError(\"Only support 2, 4, 8 bits quantization\")\n    if num_iter <= 0:\n        raise ValueError(\"Number of iterations must be greater than 0\")\n\n    out_feature, in_feature = weight.size()\n    device = weight.device\n    dtype = weight.dtype\n\n    logging.info(\n        f\"Weight: ({out_feature}, {in_feature}) | Rank: {reduced_rank} \"\n        f\"| Num Iter: {num_iter} | Num Bits: {num_bits}\"\n    )\n    if not is_bnb_4bit_available() or num_bits in [2, 8]:\n        quantizer = NFQuantizer(num_bits=num_bits, device=device, method=\"normal\", block_size=64)\n        compute_device = device\n    else:\n        compute_device = \"cuda\"\n\n    weight = weight.to(device=compute_device, dtype=torch.float32)\n    res = weight.clone()\n    for i in range(num_iter):\n        torch.cuda.empty_cache()\n        # Quantization\n        if num_bits == 4 and is_bnb_4bit_available():\n            qweight = bnb.nn.Params4bit(\n                res.to(\"cpu\"), requires_grad=False, compress_statistics=False, quant_type=\"nf4\"\n            ).to(compute_device)\n            dequantized_weight = bnb.functional.dequantize_4bit(qweight.data, qweight.quant_state)\n        else:\n            quantized_weight, max_abs, shape = quantizer.quantize_block(res)\n            dequantized_weight = quantizer.dequantize_block(quantized_weight, max_abs, shape)\n\n        res = weight - dequantized_weight\n\n        # Decompose the residual by SVD\n        output = _low_rank_decomposition(res, reduced_rank=reduced_rank)\n        L, R, reduced_rank = output[\"L\"], output[\"R\"], output[\"reduced_rank\"]\n        res = weight - torch.mm(L, R)\n\n    lora_A, lora_B = R, L\n\n    return dequantized_weight.to(device=device, dtype=dtype), lora_A, lora_B\n", "path": "src/peft/utils/loftq_utils.py"}]}