We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Feature request: Enable activation of `hf_transfer` if available.
A suggestion is to accelerate the usage of `from huggingface_hub import snapshot_download` by using `pip install hf_transfer` https://github.com/huggingface/hf_transfer

hf_transfer is a pretty lightweight binary, that uses rust to download files where python is compule / gil bound while downloading files.

Benefits: download from 200Mbit -> 2GBit (my experience in gcp, us-central) + nice to have if you have to do multiple concurrent downloads (lora)

```python
try:
    # enable hf hub transfer if available
    import hf_transfer  # type: ignore # noqa
    import huggingface_hub.constants  # type: ignore

    # can also be set  via env  var HF_HUB_ENABLE_HF_TRANSFER 
    # I would not suggest doing so, as its unclear if any venv will have 
    # pip install hf_transfer
    huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
except ImportError:
    pass
```

Let me know if this proposal is acceptable, if so, I can open a PR.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/model_executor/weight_utils.py`
Content:
```
1 """Utilities for downloading and initializing model weights."""
2 import fnmatch
3 import glob
4 import hashlib
5 import json
6 import os
7 from collections import defaultdict
8 from typing import Any, Iterator, List, Optional, Tuple
9 
10 import filelock
11 import numpy as np
12 import torch
13 from huggingface_hub import HfFileSystem, snapshot_download
14 from safetensors.torch import load_file, safe_open, save_file
15 from tqdm.auto import tqdm
16 
17 from vllm.config import ModelConfig
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.quantization import (QuantizationConfig,
20                                                      get_quantization_config)
21 
22 logger = init_logger(__name__)
23 
24 # use system-level temp directory for file locks, so that multiple users
25 # can share the same lock without error.
26 # lock files in the temp directory will be automatically deleted when the
27 # system reboots, so users will not complain about annoying lock files
28 temp_dir = os.environ.get('TMPDIR') or os.environ.get(
29     'TEMP') or os.environ.get('TMP') or "/tmp/"
30 
31 
32 class Disabledtqdm(tqdm):
33 
34     def __init__(self, *args, **kwargs):
35         super().__init__(*args, **kwargs, disable=True)
36 
37 
38 def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
39     lock_dir = cache_dir or temp_dir
40     os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
41     model_name = model_name_or_path.replace("/", "-")
42     hash_name = hashlib.sha256(model_name.encode()).hexdigest()
43     # add hash to avoid conflict with old users' lock files
44     lock_file_name = hash_name + model_name + ".lock"
45     # mode 0o666 is required for the filelock to be shared across users
46     lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),
47                              mode=0o666)
48     return lock
49 
50 
51 def _shared_pointers(tensors):
52     ptrs = defaultdict(list)
53     for k, v in tensors.items():
54         ptrs[v.data_ptr()].append(k)
55     failing = []
56     for _, names in ptrs.items():
57         if len(names) > 1:
58             failing.append(names)
59     return failing
60 
61 
62 def convert_bin_to_safetensor_file(
63     pt_filename: str,
64     sf_filename: str,
65 ) -> None:
66     loaded = torch.load(pt_filename, map_location="cpu")
67     if "state_dict" in loaded:
68         loaded = loaded["state_dict"]
69     shared = _shared_pointers(loaded)
70     for shared_weights in shared:
71         for name in shared_weights[1:]:
72             loaded.pop(name)
73 
74     # For tensors to be contiguous
75     loaded = {k: v.contiguous() for k, v in loaded.items()}
76 
77     dirname = os.path.dirname(sf_filename)
78     os.makedirs(dirname, exist_ok=True)
79     save_file(loaded, sf_filename, metadata={"format": "pt"})
80 
81     # check file size
82     sf_size = os.stat(sf_filename).st_size
83     pt_size = os.stat(pt_filename).st_size
84     if (sf_size - pt_size) / pt_size > 0.01:
85         raise RuntimeError(f"""The file size different is more than 1%:
86          - {sf_filename}: {sf_size}
87          - {pt_filename}: {pt_size}
88          """)
89 
90     # check if the tensors are the same
91     reloaded = load_file(sf_filename)
92     for k in loaded:
93         pt_tensor = loaded[k]
94         sf_tensor = reloaded[k]
95         if not torch.equal(pt_tensor, sf_tensor):
96             raise RuntimeError(f"The output tensors do not match for key {k}")
97 
98 
99 # TODO(woosuk): Move this to other place.
100 def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
101     quant_cls = get_quantization_config(model_config.quantization)
102     # Read the quantization config from the HF model config, if available.
103     hf_quant_config = getattr(model_config.hf_config, "quantization_config",
104                               None)
105     if hf_quant_config is not None:
106         return quant_cls.from_config(hf_quant_config)
107     model_name_or_path = model_config.model
108     is_local = os.path.isdir(model_name_or_path)
109     if not is_local:
110         # Download the config files.
111         with get_lock(model_name_or_path, model_config.download_dir):
112             hf_folder = snapshot_download(model_name_or_path,
113                                           revision=model_config.revision,
114                                           allow_patterns="*.json",
115                                           cache_dir=model_config.download_dir,
116                                           tqdm_class=Disabledtqdm)
117     else:
118         hf_folder = model_name_or_path
119     config_files = glob.glob(os.path.join(hf_folder, "*.json"))
120 
121     quant_config_files = [
122         f for f in config_files if any(
123             f.endswith(x) for x in quant_cls.get_config_filenames())
124     ]
125     if len(quant_config_files) == 0:
126         raise ValueError(
127             f"Cannot find the config file for {model_config.quantization}")
128     if len(quant_config_files) > 1:
129         raise ValueError(
130             f"Found multiple config files for {model_config.quantization}: "
131             f"{quant_config_files}")
132 
133     quant_config_file = quant_config_files[0]
134     with open(quant_config_file, "r") as f:
135         config = json.load(f)
136     return quant_cls.from_config(config)
137 
138 
139 def prepare_hf_model_weights(
140     model_name_or_path: str,
141     cache_dir: Optional[str] = None,
142     load_format: str = "auto",
143     fall_back_to_pt: bool = True,
144     revision: Optional[str] = None,
145 ) -> Tuple[str, List[str], bool]:
146     # Download model weights from huggingface.
147     is_local = os.path.isdir(model_name_or_path)
148     use_safetensors = False
149     # Some quantized models use .pt files for storing the weights.
150     if load_format == "auto":
151         allow_patterns = ["*.safetensors", "*.bin"]
152     elif load_format == "safetensors":
153         use_safetensors = True
154         allow_patterns = ["*.safetensors"]
155     elif load_format == "pt":
156         allow_patterns = ["*.pt"]
157     elif load_format == "npcache":
158         allow_patterns = ["*.bin"]
159     else:
160         raise ValueError(f"Unknown load_format: {load_format}")
161 
162     if fall_back_to_pt:
163         allow_patterns += ["*.pt"]
164 
165     if not is_local:
166         # Before we download we look at that is available:
167         fs = HfFileSystem()
168         file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
169 
170         # depending on what is available we download different things
171         for pattern in allow_patterns:
172             matching = fnmatch.filter(file_list, pattern)
173             if len(matching) > 0:
174                 allow_patterns = [pattern]
175                 break
176 
177         logger.info(f"Using model weights format {allow_patterns}")
178         # Use file lock to prevent multiple processes from
179         # downloading the same model weights at the same time.
180         with get_lock(model_name_or_path, cache_dir):
181             hf_folder = snapshot_download(model_name_or_path,
182                                           allow_patterns=allow_patterns,
183                                           cache_dir=cache_dir,
184                                           tqdm_class=Disabledtqdm,
185                                           revision=revision)
186     else:
187         hf_folder = model_name_or_path
188     hf_weights_files: List[str] = []
189     for pattern in allow_patterns:
190         hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
191         if len(hf_weights_files) > 0:
192             if pattern == "*.safetensors":
193                 use_safetensors = True
194             break
195     if not use_safetensors:
196         # Exclude files that are not needed for inference.
197         # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
198         blacklist = [
199             "training_args.bin",
200             "optimizer.bin",
201             "optimizer.pt",
202             "scheduler.pt",
203             "scaler.pt",
204         ]
205         hf_weights_files = [
206             f for f in hf_weights_files
207             if not any(f.endswith(x) for x in blacklist)
208         ]
209 
210     if len(hf_weights_files) == 0:
211         raise RuntimeError(
212             f"Cannot find any model weights with `{model_name_or_path}`")
213 
214     return hf_folder, hf_weights_files, use_safetensors
215 
216 
217 def hf_model_weights_iterator(
218     model_name_or_path: str,
219     cache_dir: Optional[str] = None,
220     load_format: str = "auto",
221     revision: Optional[str] = None,
222     fall_back_to_pt: Optional[bool] = True,
223 ) -> Iterator[Tuple[str, torch.Tensor]]:
224     hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
225         model_name_or_path,
226         cache_dir=cache_dir,
227         load_format=load_format,
228         fall_back_to_pt=fall_back_to_pt,
229         revision=revision)
230 
231     if load_format == "npcache":
232         # Currently np_cache only support *.bin checkpoints
233         assert use_safetensors is False
234 
235         # Convert the model weights from torch tensors to numpy arrays for
236         # faster loading.
237         np_folder = os.path.join(hf_folder, "np")
238         os.makedirs(np_folder, exist_ok=True)
239         weight_names_file = os.path.join(np_folder, "weight_names.json")
240         # Use file lock to prevent multiple processes from
241         # dumping the same model weights to numpy at the same time.
242         with get_lock(model_name_or_path, cache_dir):
243             if not os.path.exists(weight_names_file):
244                 weight_names = []
245                 for bin_file in hf_weights_files:
246                     state = torch.load(bin_file, map_location="cpu")
247                     for name, param in state.items():
248                         param_path = os.path.join(np_folder, name)
249                         with open(param_path, "wb") as f:
250                             np.save(f, param.cpu().detach().numpy())
251                         weight_names.append(name)
252                 with open(weight_names_file, "w") as f:
253                     json.dump(weight_names, f)
254 
255         with open(weight_names_file, "r") as f:
256             weight_names = json.load(f)
257 
258         for name in weight_names:
259             param_path = os.path.join(np_folder, name)
260             with open(param_path, "rb") as f:
261                 param = np.load(f)
262             yield name, torch.from_numpy(param)
263     elif use_safetensors:
264         for st_file in hf_weights_files:
265             with safe_open(st_file, framework="pt") as f:
266                 for name in f.keys():  # noqa: SIM118
267                     param = f.get_tensor(name)
268                     yield name, param
269     else:
270         for bin_file in hf_weights_files:
271             state = torch.load(bin_file, map_location="cpu")
272             for name, param in state.items():
273                 yield name, param
274             del state
275             torch.cuda.empty_cache()
276 
277 
278 def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
279     """convert PySafeSlice object from safetensors to torch.Tensor
280 
281     PySafeSlice object supports indexing, which is done before loading the
282     actual tensor and can reduce the amount of memory being read into the
283     memory. However, it does not support more advanced functionalities
284     like `.view()` or `.t()`. Therefore, if we need to modify the loaded
285     tensor with these more complicated operators, we need to convert to
286     tensor first.
287     """
288     if not isinstance(x, torch.Tensor):
289         x = x[:]
290     return x
291 
292 
293 def default_weight_loader(param: torch.Tensor,
294                           loaded_weight: torch.Tensor) -> None:
295     """Default weight loader."""
296     assert param.size() == loaded_weight.size()
297     param.data.copy_(loaded_weight)
298 
299 
300 def initialize_dummy_weights(
301     model: torch.nn.Module,
302     low: float = -1e-3,
303     high: float = 1e-3,
304 ) -> None:
305     """Initialize model weights with random values.
306 
307     The model weights must be randomly initialized for accurate performance
308     measurements. Additionally, the model weights should not cause NaNs in the
309     forward pass. We empirically found that initializing the weights with
310     values between -1e-3 and 1e-3 works well for most models.
311     """
312     for param in model.state_dict().values():
313         if torch.is_floating_point(param):
314             param.data.uniform_(low, high)
315
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py
--- a/vllm/model_executor/weight_utils.py
+++ b/vllm/model_executor/weight_utils.py
@@ -8,6 +8,7 @@
 from typing import Any, Iterator, List, Optional, Tuple
 
 import filelock
+import huggingface_hub.constants
 import numpy as np
 import torch
 from huggingface_hub import HfFileSystem, snapshot_download
@@ -29,6 +30,21 @@
     'TEMP') or os.environ.get('TMP') or "/tmp/"
 
 
+def enable_hf_transfer():
+    """automatically activates hf_transfer
+    """
+    if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ:
+        try:
+            # enable hf hub transfer if available
+            import hf_transfer  # type: ignore # noqa
+            huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True
+        except ImportError:
+            pass
+
+
+enable_hf_transfer()
+
+
 class Disabledtqdm(tqdm):
 
     def __init__(self, *args, **kwargs):

{"golden_diff": "diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py\n--- a/vllm/model_executor/weight_utils.py\n+++ b/vllm/model_executor/weight_utils.py\n@@ -8,6 +8,7 @@\n from typing import Any, Iterator, List, Optional, Tuple\n \n import filelock\n+import huggingface_hub.constants\n import numpy as np\n import torch\n from huggingface_hub import HfFileSystem, snapshot_download\n@@ -29,6 +30,21 @@\n     'TEMP') or os.environ.get('TMP') or \"/tmp/\"\n \n \n+def enable_hf_transfer():\n+    \"\"\"automatically activates hf_transfer\n+    \"\"\"\n+    if \"HF_HUB_ENABLE_HF_TRANSFER\" not in os.environ:\n+        try:\n+            # enable hf hub transfer if available\n+            import hf_transfer  # type: ignore # noqa\n+            huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True\n+        except ImportError:\n+            pass\n+\n+\n+enable_hf_transfer()\n+\n+\n class Disabledtqdm(tqdm):\n \n     def __init__(self, *args, **kwargs):\n", "issue": "Feature request: Enable activation of `hf_transfer` if available.\nA suggestion is to accelerate the usage of `from huggingface_hub import snapshot_download` by using `pip install hf_transfer` https://github.com/huggingface/hf_transfer\r\n\r\nhf_transfer is a pretty lightweight binary, that uses rust to download files where python is compule / gil bound while downloading files.\r\n\r\nBenefits: download from 200Mbit -> 2GBit (my experience in gcp, us-central) + nice to have if you have to do multiple concurrent downloads (lora)\r\n\r\n```python\r\ntry:\r\n    # enable hf hub transfer if available\r\n    import hf_transfer  # type: ignore # noqa\r\n    import huggingface_hub.constants  # type: ignore\r\n\r\n    # can also be set  via env  var HF_HUB_ENABLE_HF_TRANSFER \r\n    # I would not suggest doing so, as its unclear if any venv will have \r\n    # pip install hf_transfer\r\n    huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True\r\nexcept ImportError:\r\n    pass\r\n```\r\n\r\nLet me know if this proposal is acceptable, if so, I can open a PR.\n", "before_files": [{"content": "\"\"\"Utilities for downloading and initializing model weights.\"\"\"\nimport fnmatch\nimport glob\nimport hashlib\nimport json\nimport os\nfrom collections import defaultdict\nfrom typing import Any, Iterator, List, Optional, Tuple\n\nimport filelock\nimport numpy as np\nimport torch\nfrom huggingface_hub import HfFileSystem, snapshot_download\nfrom safetensors.torch import load_file, safe_open, save_file\nfrom tqdm.auto import tqdm\n\nfrom vllm.config import ModelConfig\nfrom vllm.logger import init_logger\nfrom vllm.model_executor.layers.quantization import (QuantizationConfig,\n                                                     get_quantization_config)\n\nlogger = init_logger(__name__)\n\n# use system-level temp directory for file locks, so that multiple users\n# can share the same lock without error.\n# lock files in the temp directory will be automatically deleted when the\n# system reboots, so users will not complain about annoying lock files\ntemp_dir = os.environ.get('TMPDIR') or os.environ.get(\n    'TEMP') or os.environ.get('TMP') or \"/tmp/\"\n\n\nclass Disabledtqdm(tqdm):\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs, disable=True)\n\n\ndef get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):\n    lock_dir = cache_dir or temp_dir\n    os.makedirs(os.path.dirname(lock_dir), exist_ok=True)\n    model_name = model_name_or_path.replace(\"/\", \"-\")\n    hash_name = hashlib.sha256(model_name.encode()).hexdigest()\n    # add hash to avoid conflict with old users' lock files\n    lock_file_name = hash_name + model_name + \".lock\"\n    # mode 0o666 is required for the filelock to be shared across users\n    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),\n                             mode=0o666)\n    return lock\n\n\ndef _shared_pointers(tensors):\n    ptrs = defaultdict(list)\n    for k, v in tensors.items():\n        ptrs[v.data_ptr()].append(k)\n    failing = []\n    for _, names in ptrs.items():\n        if len(names) > 1:\n            failing.append(names)\n    return failing\n\n\ndef convert_bin_to_safetensor_file(\n    pt_filename: str,\n    sf_filename: str,\n) -> None:\n    loaded = torch.load(pt_filename, map_location=\"cpu\")\n    if \"state_dict\" in loaded:\n        loaded = loaded[\"state_dict\"]\n    shared = _shared_pointers(loaded)\n    for shared_weights in shared:\n        for name in shared_weights[1:]:\n            loaded.pop(name)\n\n    # For tensors to be contiguous\n    loaded = {k: v.contiguous() for k, v in loaded.items()}\n\n    dirname = os.path.dirname(sf_filename)\n    os.makedirs(dirname, exist_ok=True)\n    save_file(loaded, sf_filename, metadata={\"format\": \"pt\"})\n\n    # check file size\n    sf_size = os.stat(sf_filename).st_size\n    pt_size = os.stat(pt_filename).st_size\n    if (sf_size - pt_size) / pt_size > 0.01:\n        raise RuntimeError(f\"\"\"The file size different is more than 1%:\n         - {sf_filename}: {sf_size}\n         - {pt_filename}: {pt_size}\n         \"\"\")\n\n    # check if the tensors are the same\n    reloaded = load_file(sf_filename)\n    for k in loaded:\n        pt_tensor = loaded[k]\n        sf_tensor = reloaded[k]\n        if not torch.equal(pt_tensor, sf_tensor):\n            raise RuntimeError(f\"The output tensors do not match for key {k}\")\n\n\n# TODO(woosuk): Move this to other place.\ndef get_quant_config(model_config: ModelConfig) -> QuantizationConfig:\n    quant_cls = get_quantization_config(model_config.quantization)\n    # Read the quantization config from the HF model config, if available.\n    hf_quant_config = getattr(model_config.hf_config, \"quantization_config\",\n                              None)\n    if hf_quant_config is not None:\n        return quant_cls.from_config(hf_quant_config)\n    model_name_or_path = model_config.model\n    is_local = os.path.isdir(model_name_or_path)\n    if not is_local:\n        # Download the config files.\n        with get_lock(model_name_or_path, model_config.download_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          revision=model_config.revision,\n                                          allow_patterns=\"*.json\",\n                                          cache_dir=model_config.download_dir,\n                                          tqdm_class=Disabledtqdm)\n    else:\n        hf_folder = model_name_or_path\n    config_files = glob.glob(os.path.join(hf_folder, \"*.json\"))\n\n    quant_config_files = [\n        f for f in config_files if any(\n            f.endswith(x) for x in quant_cls.get_config_filenames())\n    ]\n    if len(quant_config_files) == 0:\n        raise ValueError(\n            f\"Cannot find the config file for {model_config.quantization}\")\n    if len(quant_config_files) > 1:\n        raise ValueError(\n            f\"Found multiple config files for {model_config.quantization}: \"\n            f\"{quant_config_files}\")\n\n    quant_config_file = quant_config_files[0]\n    with open(quant_config_file, \"r\") as f:\n        config = json.load(f)\n    return quant_cls.from_config(config)\n\n\ndef prepare_hf_model_weights(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    fall_back_to_pt: bool = True,\n    revision: Optional[str] = None,\n) -> Tuple[str, List[str], bool]:\n    # Download model weights from huggingface.\n    is_local = os.path.isdir(model_name_or_path)\n    use_safetensors = False\n    # Some quantized models use .pt files for storing the weights.\n    if load_format == \"auto\":\n        allow_patterns = [\"*.safetensors\", \"*.bin\"]\n    elif load_format == \"safetensors\":\n        use_safetensors = True\n        allow_patterns = [\"*.safetensors\"]\n    elif load_format == \"pt\":\n        allow_patterns = [\"*.pt\"]\n    elif load_format == \"npcache\":\n        allow_patterns = [\"*.bin\"]\n    else:\n        raise ValueError(f\"Unknown load_format: {load_format}\")\n\n    if fall_back_to_pt:\n        allow_patterns += [\"*.pt\"]\n\n    if not is_local:\n        # Before we download we look at that is available:\n        fs = HfFileSystem()\n        file_list = fs.ls(model_name_or_path, detail=False, revision=revision)\n\n        # depending on what is available we download different things\n        for pattern in allow_patterns:\n            matching = fnmatch.filter(file_list, pattern)\n            if len(matching) > 0:\n                allow_patterns = [pattern]\n                break\n\n        logger.info(f\"Using model weights format {allow_patterns}\")\n        # Use file lock to prevent multiple processes from\n        # downloading the same model weights at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          allow_patterns=allow_patterns,\n                                          cache_dir=cache_dir,\n                                          tqdm_class=Disabledtqdm,\n                                          revision=revision)\n    else:\n        hf_folder = model_name_or_path\n    hf_weights_files: List[str] = []\n    for pattern in allow_patterns:\n        hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))\n        if len(hf_weights_files) > 0:\n            if pattern == \"*.safetensors\":\n                use_safetensors = True\n            break\n    if not use_safetensors:\n        # Exclude files that are not needed for inference.\n        # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233\n        blacklist = [\n            \"training_args.bin\",\n            \"optimizer.bin\",\n            \"optimizer.pt\",\n            \"scheduler.pt\",\n            \"scaler.pt\",\n        ]\n        hf_weights_files = [\n            f for f in hf_weights_files\n            if not any(f.endswith(x) for x in blacklist)\n        ]\n\n    if len(hf_weights_files) == 0:\n        raise RuntimeError(\n            f\"Cannot find any model weights with `{model_name_or_path}`\")\n\n    return hf_folder, hf_weights_files, use_safetensors\n\n\ndef hf_model_weights_iterator(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    revision: Optional[str] = None,\n    fall_back_to_pt: Optional[bool] = True,\n) -> Iterator[Tuple[str, torch.Tensor]]:\n    hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(\n        model_name_or_path,\n        cache_dir=cache_dir,\n        load_format=load_format,\n        fall_back_to_pt=fall_back_to_pt,\n        revision=revision)\n\n    if load_format == \"npcache\":\n        # Currently np_cache only support *.bin checkpoints\n        assert use_safetensors is False\n\n        # Convert the model weights from torch tensors to numpy arrays for\n        # faster loading.\n        np_folder = os.path.join(hf_folder, \"np\")\n        os.makedirs(np_folder, exist_ok=True)\n        weight_names_file = os.path.join(np_folder, \"weight_names.json\")\n        # Use file lock to prevent multiple processes from\n        # dumping the same model weights to numpy at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            if not os.path.exists(weight_names_file):\n                weight_names = []\n                for bin_file in hf_weights_files:\n                    state = torch.load(bin_file, map_location=\"cpu\")\n                    for name, param in state.items():\n                        param_path = os.path.join(np_folder, name)\n                        with open(param_path, \"wb\") as f:\n                            np.save(f, param.cpu().detach().numpy())\n                        weight_names.append(name)\n                with open(weight_names_file, \"w\") as f:\n                    json.dump(weight_names, f)\n\n        with open(weight_names_file, \"r\") as f:\n            weight_names = json.load(f)\n\n        for name in weight_names:\n            param_path = os.path.join(np_folder, name)\n            with open(param_path, \"rb\") as f:\n                param = np.load(f)\n            yield name, torch.from_numpy(param)\n    elif use_safetensors:\n        for st_file in hf_weights_files:\n            with safe_open(st_file, framework=\"pt\") as f:\n                for name in f.keys():  # noqa: SIM118\n                    param = f.get_tensor(name)\n                    yield name, param\n    else:\n        for bin_file in hf_weights_files:\n            state = torch.load(bin_file, map_location=\"cpu\")\n            for name, param in state.items():\n                yield name, param\n            del state\n            torch.cuda.empty_cache()\n\n\ndef convert_pyslice_to_tensor(x: Any) -> torch.Tensor:\n    \"\"\"convert PySafeSlice object from safetensors to torch.Tensor\n\n    PySafeSlice object supports indexing, which is done before loading the\n    actual tensor and can reduce the amount of memory being read into the\n    memory. However, it does not support more advanced functionalities\n    like `.view()` or `.t()`. Therefore, if we need to modify the loaded\n    tensor with these more complicated operators, we need to convert to\n    tensor first.\n    \"\"\"\n    if not isinstance(x, torch.Tensor):\n        x = x[:]\n    return x\n\n\ndef default_weight_loader(param: torch.Tensor,\n                          loaded_weight: torch.Tensor) -> None:\n    \"\"\"Default weight loader.\"\"\"\n    assert param.size() == loaded_weight.size()\n    param.data.copy_(loaded_weight)\n\n\ndef initialize_dummy_weights(\n    model: torch.nn.Module,\n    low: float = -1e-3,\n    high: float = 1e-3,\n) -> None:\n    \"\"\"Initialize model weights with random values.\n\n    The model weights must be randomly initialized for accurate performance\n    measurements. Additionally, the model weights should not cause NaNs in the\n    forward pass. We empirically found that initializing the weights with\n    values between -1e-3 and 1e-3 works well for most models.\n    \"\"\"\n    for param in model.state_dict().values():\n        if torch.is_floating_point(param):\n            param.data.uniform_(low, high)\n", "path": "vllm/model_executor/weight_utils.py"}], "after_files": [{"content": "\"\"\"Utilities for downloading and initializing model weights.\"\"\"\nimport fnmatch\nimport glob\nimport hashlib\nimport json\nimport os\nfrom collections import defaultdict\nfrom typing import Any, Iterator, List, Optional, Tuple\n\nimport filelock\nimport huggingface_hub.constants\nimport numpy as np\nimport torch\nfrom huggingface_hub import HfFileSystem, snapshot_download\nfrom safetensors.torch import load_file, safe_open, save_file\nfrom tqdm.auto import tqdm\n\nfrom vllm.config import ModelConfig\nfrom vllm.logger import init_logger\nfrom vllm.model_executor.layers.quantization import (QuantizationConfig,\n                                                     get_quantization_config)\n\nlogger = init_logger(__name__)\n\n# use system-level temp directory for file locks, so that multiple users\n# can share the same lock without error.\n# lock files in the temp directory will be automatically deleted when the\n# system reboots, so users will not complain about annoying lock files\ntemp_dir = os.environ.get('TMPDIR') or os.environ.get(\n    'TEMP') or os.environ.get('TMP') or \"/tmp/\"\n\n\ndef enable_hf_transfer():\n    \"\"\"automatically activates hf_transfer\n    \"\"\"\n    if \"HF_HUB_ENABLE_HF_TRANSFER\" not in os.environ:\n        try:\n            # enable hf hub transfer if available\n            import hf_transfer  # type: ignore # noqa\n            huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True\n        except ImportError:\n            pass\n\n\nenable_hf_transfer()\n\n\nclass Disabledtqdm(tqdm):\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs, disable=True)\n\n\ndef get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):\n    lock_dir = cache_dir or temp_dir\n    os.makedirs(os.path.dirname(lock_dir), exist_ok=True)\n    model_name = model_name_or_path.replace(\"/\", \"-\")\n    hash_name = hashlib.sha256(model_name.encode()).hexdigest()\n    # add hash to avoid conflict with old users' lock files\n    lock_file_name = hash_name + model_name + \".lock\"\n    # mode 0o666 is required for the filelock to be shared across users\n    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name),\n                             mode=0o666)\n    return lock\n\n\ndef _shared_pointers(tensors):\n    ptrs = defaultdict(list)\n    for k, v in tensors.items():\n        ptrs[v.data_ptr()].append(k)\n    failing = []\n    for _, names in ptrs.items():\n        if len(names) > 1:\n            failing.append(names)\n    return failing\n\n\ndef convert_bin_to_safetensor_file(\n    pt_filename: str,\n    sf_filename: str,\n) -> None:\n    loaded = torch.load(pt_filename, map_location=\"cpu\")\n    if \"state_dict\" in loaded:\n        loaded = loaded[\"state_dict\"]\n    shared = _shared_pointers(loaded)\n    for shared_weights in shared:\n        for name in shared_weights[1:]:\n            loaded.pop(name)\n\n    # For tensors to be contiguous\n    loaded = {k: v.contiguous() for k, v in loaded.items()}\n\n    dirname = os.path.dirname(sf_filename)\n    os.makedirs(dirname, exist_ok=True)\n    save_file(loaded, sf_filename, metadata={\"format\": \"pt\"})\n\n    # check file size\n    sf_size = os.stat(sf_filename).st_size\n    pt_size = os.stat(pt_filename).st_size\n    if (sf_size - pt_size) / pt_size > 0.01:\n        raise RuntimeError(f\"\"\"The file size different is more than 1%:\n         - {sf_filename}: {sf_size}\n         - {pt_filename}: {pt_size}\n         \"\"\")\n\n    # check if the tensors are the same\n    reloaded = load_file(sf_filename)\n    for k in loaded:\n        pt_tensor = loaded[k]\n        sf_tensor = reloaded[k]\n        if not torch.equal(pt_tensor, sf_tensor):\n            raise RuntimeError(f\"The output tensors do not match for key {k}\")\n\n\n# TODO(woosuk): Move this to other place.\ndef get_quant_config(model_config: ModelConfig) -> QuantizationConfig:\n    quant_cls = get_quantization_config(model_config.quantization)\n    # Read the quantization config from the HF model config, if available.\n    hf_quant_config = getattr(model_config.hf_config, \"quantization_config\",\n                              None)\n    if hf_quant_config is not None:\n        return quant_cls.from_config(hf_quant_config)\n    model_name_or_path = model_config.model\n    is_local = os.path.isdir(model_name_or_path)\n    if not is_local:\n        # Download the config files.\n        with get_lock(model_name_or_path, model_config.download_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          revision=model_config.revision,\n                                          allow_patterns=\"*.json\",\n                                          cache_dir=model_config.download_dir,\n                                          tqdm_class=Disabledtqdm)\n    else:\n        hf_folder = model_name_or_path\n    config_files = glob.glob(os.path.join(hf_folder, \"*.json\"))\n\n    quant_config_files = [\n        f for f in config_files if any(\n            f.endswith(x) for x in quant_cls.get_config_filenames())\n    ]\n    if len(quant_config_files) == 0:\n        raise ValueError(\n            f\"Cannot find the config file for {model_config.quantization}\")\n    if len(quant_config_files) > 1:\n        raise ValueError(\n            f\"Found multiple config files for {model_config.quantization}: \"\n            f\"{quant_config_files}\")\n\n    quant_config_file = quant_config_files[0]\n    with open(quant_config_file, \"r\") as f:\n        config = json.load(f)\n    return quant_cls.from_config(config)\n\n\ndef prepare_hf_model_weights(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    fall_back_to_pt: bool = True,\n    revision: Optional[str] = None,\n) -> Tuple[str, List[str], bool]:\n    # Download model weights from huggingface.\n    is_local = os.path.isdir(model_name_or_path)\n    use_safetensors = False\n    # Some quantized models use .pt files for storing the weights.\n    if load_format == \"auto\":\n        allow_patterns = [\"*.safetensors\", \"*.bin\"]\n    elif load_format == \"safetensors\":\n        use_safetensors = True\n        allow_patterns = [\"*.safetensors\"]\n    elif load_format == \"pt\":\n        allow_patterns = [\"*.pt\"]\n    elif load_format == \"npcache\":\n        allow_patterns = [\"*.bin\"]\n    else:\n        raise ValueError(f\"Unknown load_format: {load_format}\")\n\n    if fall_back_to_pt:\n        allow_patterns += [\"*.pt\"]\n\n    if not is_local:\n        # Before we download we look at that is available:\n        fs = HfFileSystem()\n        file_list = fs.ls(model_name_or_path, detail=False, revision=revision)\n\n        # depending on what is available we download different things\n        for pattern in allow_patterns:\n            matching = fnmatch.filter(file_list, pattern)\n            if len(matching) > 0:\n                allow_patterns = [pattern]\n                break\n\n        logger.info(f\"Using model weights format {allow_patterns}\")\n        # Use file lock to prevent multiple processes from\n        # downloading the same model weights at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          allow_patterns=allow_patterns,\n                                          cache_dir=cache_dir,\n                                          tqdm_class=Disabledtqdm,\n                                          revision=revision)\n    else:\n        hf_folder = model_name_or_path\n    hf_weights_files: List[str] = []\n    for pattern in allow_patterns:\n        hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))\n        if len(hf_weights_files) > 0:\n            if pattern == \"*.safetensors\":\n                use_safetensors = True\n            break\n    if not use_safetensors:\n        # Exclude files that are not needed for inference.\n        # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233\n        blacklist = [\n            \"training_args.bin\",\n            \"optimizer.bin\",\n            \"optimizer.pt\",\n            \"scheduler.pt\",\n            \"scaler.pt\",\n        ]\n        hf_weights_files = [\n            f for f in hf_weights_files\n            if not any(f.endswith(x) for x in blacklist)\n        ]\n\n    if len(hf_weights_files) == 0:\n        raise RuntimeError(\n            f\"Cannot find any model weights with `{model_name_or_path}`\")\n\n    return hf_folder, hf_weights_files, use_safetensors\n\n\ndef hf_model_weights_iterator(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    revision: Optional[str] = None,\n    fall_back_to_pt: Optional[bool] = True,\n) -> Iterator[Tuple[str, torch.Tensor]]:\n    hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(\n        model_name_or_path,\n        cache_dir=cache_dir,\n        load_format=load_format,\n        fall_back_to_pt=fall_back_to_pt,\n        revision=revision)\n\n    if load_format == \"npcache\":\n        # Currently np_cache only support *.bin checkpoints\n        assert use_safetensors is False\n\n        # Convert the model weights from torch tensors to numpy arrays for\n        # faster loading.\n        np_folder = os.path.join(hf_folder, \"np\")\n        os.makedirs(np_folder, exist_ok=True)\n        weight_names_file = os.path.join(np_folder, \"weight_names.json\")\n        # Use file lock to prevent multiple processes from\n        # dumping the same model weights to numpy at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            if not os.path.exists(weight_names_file):\n                weight_names = []\n                for bin_file in hf_weights_files:\n                    state = torch.load(bin_file, map_location=\"cpu\")\n                    for name, param in state.items():\n                        param_path = os.path.join(np_folder, name)\n                        with open(param_path, \"wb\") as f:\n                            np.save(f, param.cpu().detach().numpy())\n                        weight_names.append(name)\n                with open(weight_names_file, \"w\") as f:\n                    json.dump(weight_names, f)\n\n        with open(weight_names_file, \"r\") as f:\n            weight_names = json.load(f)\n\n        for name in weight_names:\n            param_path = os.path.join(np_folder, name)\n            with open(param_path, \"rb\") as f:\n                param = np.load(f)\n            yield name, torch.from_numpy(param)\n    elif use_safetensors:\n        for st_file in hf_weights_files:\n            with safe_open(st_file, framework=\"pt\") as f:\n                for name in f.keys():  # noqa: SIM118\n                    param = f.get_tensor(name)\n                    yield name, param\n    else:\n        for bin_file in hf_weights_files:\n            state = torch.load(bin_file, map_location=\"cpu\")\n            for name, param in state.items():\n                yield name, param\n            del state\n            torch.cuda.empty_cache()\n\n\ndef convert_pyslice_to_tensor(x: Any) -> torch.Tensor:\n    \"\"\"convert PySafeSlice object from safetensors to torch.Tensor\n\n    PySafeSlice object supports indexing, which is done before loading the\n    actual tensor and can reduce the amount of memory being read into the\n    memory. However, it does not support more advanced functionalities\n    like `.view()` or `.t()`. Therefore, if we need to modify the loaded\n    tensor with these more complicated operators, we need to convert to\n    tensor first.\n    \"\"\"\n    if not isinstance(x, torch.Tensor):\n        x = x[:]\n    return x\n\n\ndef default_weight_loader(param: torch.Tensor,\n                          loaded_weight: torch.Tensor) -> None:\n    \"\"\"Default weight loader.\"\"\"\n    assert param.size() == loaded_weight.size()\n    param.data.copy_(loaded_weight)\n\n\ndef initialize_dummy_weights(\n    model: torch.nn.Module,\n    low: float = -1e-3,\n    high: float = 1e-3,\n) -> None:\n    \"\"\"Initialize model weights with random values.\n\n    The model weights must be randomly initialized for accurate performance\n    measurements. Additionally, the model weights should not cause NaNs in the\n    forward pass. We empirically found that initializing the weights with\n    values between -1e-3 and 1e-3 works well for most models.\n    \"\"\"\n    for param in model.state_dict().values():\n        if torch.is_floating_point(param):\n            param.data.uniform_(low, high)\n", "path": "vllm/model_executor/weight_utils.py"}]}