We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
The same model cannot be loaded by two different users
As pointed out here, the way lockfiles are created prevents the second user from loading any models that a previous user has loaded at any point: https://github.com/vllm-project/vllm/issues/2179

This is still an issue with the only workaround being to force-delete the lockfile created by another user.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/model_executor/weight_utils.py`
Content:
```
1 """Utilities for downloading and initializing model weights."""
2 import filelock
3 import glob
4 import fnmatch
5 import json
6 import os
7 from collections import defaultdict
8 from typing import Any, Iterator, List, Optional, Tuple
9 
10 from huggingface_hub import snapshot_download, HfFileSystem
11 import numpy as np
12 from safetensors.torch import load_file, save_file, safe_open
13 import torch
14 from tqdm.auto import tqdm
15 
16 from vllm.config import ModelConfig
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.quantization import (get_quantization_config,
19                                                      QuantizationConfig)
20 
21 logger = init_logger(__name__)
22 
23 _xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
24 _vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')
25 
26 
27 class Disabledtqdm(tqdm):
28 
29     def __init__(self, *args, **kwargs):
30         super().__init__(*args, **kwargs, disable=True)
31 
32 
33 def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
34     lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
35     os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
36     lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
37     lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
38     return lock
39 
40 
41 def _shared_pointers(tensors):
42     ptrs = defaultdict(list)
43     for k, v in tensors.items():
44         ptrs[v.data_ptr()].append(k)
45     failing = []
46     for _, names in ptrs.items():
47         if len(names) > 1:
48             failing.append(names)
49     return failing
50 
51 
52 def convert_bin_to_safetensor_file(
53     pt_filename: str,
54     sf_filename: str,
55 ) -> None:
56     loaded = torch.load(pt_filename, map_location="cpu")
57     if "state_dict" in loaded:
58         loaded = loaded["state_dict"]
59     shared = _shared_pointers(loaded)
60     for shared_weights in shared:
61         for name in shared_weights[1:]:
62             loaded.pop(name)
63 
64     # For tensors to be contiguous
65     loaded = {k: v.contiguous() for k, v in loaded.items()}
66 
67     dirname = os.path.dirname(sf_filename)
68     os.makedirs(dirname, exist_ok=True)
69     save_file(loaded, sf_filename, metadata={"format": "pt"})
70 
71     # check file size
72     sf_size = os.stat(sf_filename).st_size
73     pt_size = os.stat(pt_filename).st_size
74     if (sf_size - pt_size) / pt_size > 0.01:
75         raise RuntimeError(f"""The file size different is more than 1%:
76          - {sf_filename}: {sf_size}
77          - {pt_filename}: {pt_size}
78          """)
79 
80     # check if the tensors are the same
81     reloaded = load_file(sf_filename)
82     for k in loaded:
83         pt_tensor = loaded[k]
84         sf_tensor = reloaded[k]
85         if not torch.equal(pt_tensor, sf_tensor):
86             raise RuntimeError(f"The output tensors do not match for key {k}")
87 
88 
89 # TODO(woosuk): Move this to other place.
90 def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
91     quant_cls = get_quantization_config(model_config.quantization)
92     # Read the quantization config from the HF model config, if available.
93     hf_quant_config = getattr(model_config.hf_config, "quantization_config",
94                               None)
95     if hf_quant_config is not None:
96         return quant_cls.from_config(hf_quant_config)
97     model_name_or_path = model_config.model
98     is_local = os.path.isdir(model_name_or_path)
99     if not is_local:
100         # Download the config files.
101         with get_lock(model_name_or_path, model_config.download_dir):
102             hf_folder = snapshot_download(model_name_or_path,
103                                           revision=model_config.revision,
104                                           allow_patterns="*.json",
105                                           cache_dir=model_config.download_dir,
106                                           tqdm_class=Disabledtqdm)
107     else:
108         hf_folder = model_name_or_path
109     config_files = glob.glob(os.path.join(hf_folder, "*.json"))
110 
111     quant_config_files = [
112         f for f in config_files if any(
113             f.endswith(x) for x in quant_cls.get_config_filenames())
114     ]
115     if len(quant_config_files) == 0:
116         raise ValueError(
117             f"Cannot find the config file for {model_config.quantization}")
118     if len(quant_config_files) > 1:
119         raise ValueError(
120             f"Found multiple config files for {model_config.quantization}: "
121             f"{quant_config_files}")
122 
123     quant_config_file = quant_config_files[0]
124     with open(quant_config_file, "r") as f:
125         config = json.load(f)
126     return quant_cls.from_config(config)
127 
128 
129 def prepare_hf_model_weights(
130     model_name_or_path: str,
131     cache_dir: Optional[str] = None,
132     load_format: str = "auto",
133     fall_back_to_pt: bool = True,
134     revision: Optional[str] = None,
135 ) -> Tuple[str, List[str], bool]:
136     # Download model weights from huggingface.
137     is_local = os.path.isdir(model_name_or_path)
138     use_safetensors = False
139     # Some quantized models use .pt files for storing the weights.
140     if load_format == "auto":
141         allow_patterns = ["*.safetensors", "*.bin"]
142     elif load_format == "safetensors":
143         use_safetensors = True
144         allow_patterns = ["*.safetensors"]
145     elif load_format == "pt":
146         allow_patterns = ["*.pt"]
147     elif load_format == "npcache":
148         allow_patterns = ["*.bin"]
149     else:
150         raise ValueError(f"Unknown load_format: {load_format}")
151 
152     if fall_back_to_pt:
153         allow_patterns += ["*.pt"]
154 
155     if not is_local:
156         # Before we download we look at that is available:
157         fs = HfFileSystem()
158         file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
159 
160         # depending on what is available we download different things
161         for pattern in allow_patterns:
162             matching = fnmatch.filter(file_list, pattern)
163             if len(matching) > 0:
164                 allow_patterns = [pattern]
165                 break
166 
167         logger.info(f"Using model weights format {allow_patterns}")
168         # Use file lock to prevent multiple processes from
169         # downloading the same model weights at the same time.
170         with get_lock(model_name_or_path, cache_dir):
171             hf_folder = snapshot_download(model_name_or_path,
172                                           allow_patterns=allow_patterns,
173                                           cache_dir=cache_dir,
174                                           tqdm_class=Disabledtqdm,
175                                           revision=revision)
176     else:
177         hf_folder = model_name_or_path
178     hf_weights_files: List[str] = []
179     for pattern in allow_patterns:
180         hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
181         if len(hf_weights_files) > 0:
182             if pattern == "*.safetensors":
183                 use_safetensors = True
184             break
185     if not use_safetensors:
186         # Exclude files that are not needed for inference.
187         # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
188         blacklist = [
189             "training_args.bin",
190             "optimizer.bin",
191             "optimizer.pt",
192             "scheduler.pt",
193             "scaler.pt",
194         ]
195         hf_weights_files = [
196             f for f in hf_weights_files
197             if not any(f.endswith(x) for x in blacklist)
198         ]
199 
200     if len(hf_weights_files) == 0:
201         raise RuntimeError(
202             f"Cannot find any model weights with `{model_name_or_path}`")
203 
204     return hf_folder, hf_weights_files, use_safetensors
205 
206 
207 def hf_model_weights_iterator(
208     model_name_or_path: str,
209     cache_dir: Optional[str] = None,
210     load_format: str = "auto",
211     revision: Optional[str] = None,
212     fall_back_to_pt: Optional[bool] = True,
213 ) -> Iterator[Tuple[str, torch.Tensor]]:
214     hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
215         model_name_or_path,
216         cache_dir=cache_dir,
217         load_format=load_format,
218         fall_back_to_pt=fall_back_to_pt,
219         revision=revision)
220 
221     if load_format == "npcache":
222         # Currently np_cache only support *.bin checkpoints
223         assert use_safetensors is False
224 
225         # Convert the model weights from torch tensors to numpy arrays for
226         # faster loading.
227         np_folder = os.path.join(hf_folder, "np")
228         os.makedirs(np_folder, exist_ok=True)
229         weight_names_file = os.path.join(np_folder, "weight_names.json")
230         # Use file lock to prevent multiple processes from
231         # dumping the same model weights to numpy at the same time.
232         with get_lock(model_name_or_path, cache_dir):
233             if not os.path.exists(weight_names_file):
234                 weight_names = []
235                 for bin_file in hf_weights_files:
236                     state = torch.load(bin_file, map_location="cpu")
237                     for name, param in state.items():
238                         param_path = os.path.join(np_folder, name)
239                         with open(param_path, "wb") as f:
240                             np.save(f, param.cpu().detach().numpy())
241                         weight_names.append(name)
242                 with open(weight_names_file, "w") as f:
243                     json.dump(weight_names, f)
244 
245         with open(weight_names_file, "r") as f:
246             weight_names = json.load(f)
247 
248         for name in weight_names:
249             param_path = os.path.join(np_folder, name)
250             with open(param_path, "rb") as f:
251                 param = np.load(f)
252             yield name, torch.from_numpy(param)
253     elif use_safetensors:
254         for st_file in hf_weights_files:
255             with safe_open(st_file, framework="pt") as f:
256                 for name in f.keys():  # noqa: SIM118
257                     param = f.get_tensor(name)
258                     yield name, param
259     else:
260         for bin_file in hf_weights_files:
261             state = torch.load(bin_file, map_location="cpu")
262             for name, param in state.items():
263                 yield name, param
264             del state
265             torch.cuda.empty_cache()
266 
267 
268 def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
269     """convert PySafeSlice object from safetensors to torch.Tensor
270 
271     PySafeSlice object supports indexing, which is done before loading the
272     actual tensor and can reduce the amount of memory being read into the
273     memory. However, it does not support more advanced functionalities
274     like `.view()` or `.t()`. Therefore, if we need to modify the loaded
275     tensor with these more complicated operators, we need to convert to
276     tensor first.
277     """
278     if not isinstance(x, torch.Tensor):
279         x = x[:]
280     return x
281 
282 
283 def default_weight_loader(param: torch.Tensor,
284                           loaded_weight: torch.Tensor) -> None:
285     """Default weight loader."""
286     assert param.size() == loaded_weight.size()
287     param.data.copy_(loaded_weight)
288 
289 
290 def initialize_dummy_weights(
291     model: torch.nn.Module,
292     low: float = -1e-3,
293     high: float = 1e-3,
294 ) -> None:
295     """Initialize model weights with random values.
296 
297     The model weights must be randomly initialized for accurate performance
298     measurements. Additionally, the model weights should not cause NaNs in the
299     forward pass. We empirically found that initializing the weights with
300     values between -1e-3 and 1e-3 works well for most models.
301     """
302     for param in model.state_dict().values():
303         if torch.is_floating_point(param):
304             param.data.uniform_(low, high)
305
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py
--- a/vllm/model_executor/weight_utils.py
+++ b/vllm/model_executor/weight_utils.py
@@ -34,7 +34,7 @@
     lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
     os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
     lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
-    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
+    lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))
     return lock

{"golden_diff": "diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py\n--- a/vllm/model_executor/weight_utils.py\n+++ b/vllm/model_executor/weight_utils.py\n@@ -34,7 +34,7 @@\n     lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path\n     os.makedirs(os.path.dirname(lock_dir), exist_ok=True)\n     lock_file_name = model_name_or_path.replace(\"/\", \"-\") + \".lock\"\n-    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))\n+    lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))\n     return lock\n", "issue": "The same model cannot be loaded by two different users\nAs pointed out here, the way lockfiles are created prevents the second user from loading any models that a previous user has loaded at any point: https://github.com/vllm-project/vllm/issues/2179\r\n\r\nThis is still an issue with the only workaround being to force-delete the lockfile created by another user.\n", "before_files": [{"content": "\"\"\"Utilities for downloading and initializing model weights.\"\"\"\nimport filelock\nimport glob\nimport fnmatch\nimport json\nimport os\nfrom collections import defaultdict\nfrom typing import Any, Iterator, List, Optional, Tuple\n\nfrom huggingface_hub import snapshot_download, HfFileSystem\nimport numpy as np\nfrom safetensors.torch import load_file, save_file, safe_open\nimport torch\nfrom tqdm.auto import tqdm\n\nfrom vllm.config import ModelConfig\nfrom vllm.logger import init_logger\nfrom vllm.model_executor.layers.quantization import (get_quantization_config,\n                                                     QuantizationConfig)\n\nlogger = init_logger(__name__)\n\n_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))\n_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')\n\n\nclass Disabledtqdm(tqdm):\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs, disable=True)\n\n\ndef get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):\n    lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path\n    os.makedirs(os.path.dirname(lock_dir), exist_ok=True)\n    lock_file_name = model_name_or_path.replace(\"/\", \"-\") + \".lock\"\n    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))\n    return lock\n\n\ndef _shared_pointers(tensors):\n    ptrs = defaultdict(list)\n    for k, v in tensors.items():\n        ptrs[v.data_ptr()].append(k)\n    failing = []\n    for _, names in ptrs.items():\n        if len(names) > 1:\n            failing.append(names)\n    return failing\n\n\ndef convert_bin_to_safetensor_file(\n    pt_filename: str,\n    sf_filename: str,\n) -> None:\n    loaded = torch.load(pt_filename, map_location=\"cpu\")\n    if \"state_dict\" in loaded:\n        loaded = loaded[\"state_dict\"]\n    shared = _shared_pointers(loaded)\n    for shared_weights in shared:\n        for name in shared_weights[1:]:\n            loaded.pop(name)\n\n    # For tensors to be contiguous\n    loaded = {k: v.contiguous() for k, v in loaded.items()}\n\n    dirname = os.path.dirname(sf_filename)\n    os.makedirs(dirname, exist_ok=True)\n    save_file(loaded, sf_filename, metadata={\"format\": \"pt\"})\n\n    # check file size\n    sf_size = os.stat(sf_filename).st_size\n    pt_size = os.stat(pt_filename).st_size\n    if (sf_size - pt_size) / pt_size > 0.01:\n        raise RuntimeError(f\"\"\"The file size different is more than 1%:\n         - {sf_filename}: {sf_size}\n         - {pt_filename}: {pt_size}\n         \"\"\")\n\n    # check if the tensors are the same\n    reloaded = load_file(sf_filename)\n    for k in loaded:\n        pt_tensor = loaded[k]\n        sf_tensor = reloaded[k]\n        if not torch.equal(pt_tensor, sf_tensor):\n            raise RuntimeError(f\"The output tensors do not match for key {k}\")\n\n\n# TODO(woosuk): Move this to other place.\ndef get_quant_config(model_config: ModelConfig) -> QuantizationConfig:\n    quant_cls = get_quantization_config(model_config.quantization)\n    # Read the quantization config from the HF model config, if available.\n    hf_quant_config = getattr(model_config.hf_config, \"quantization_config\",\n                              None)\n    if hf_quant_config is not None:\n        return quant_cls.from_config(hf_quant_config)\n    model_name_or_path = model_config.model\n    is_local = os.path.isdir(model_name_or_path)\n    if not is_local:\n        # Download the config files.\n        with get_lock(model_name_or_path, model_config.download_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          revision=model_config.revision,\n                                          allow_patterns=\"*.json\",\n                                          cache_dir=model_config.download_dir,\n                                          tqdm_class=Disabledtqdm)\n    else:\n        hf_folder = model_name_or_path\n    config_files = glob.glob(os.path.join(hf_folder, \"*.json\"))\n\n    quant_config_files = [\n        f for f in config_files if any(\n            f.endswith(x) for x in quant_cls.get_config_filenames())\n    ]\n    if len(quant_config_files) == 0:\n        raise ValueError(\n            f\"Cannot find the config file for {model_config.quantization}\")\n    if len(quant_config_files) > 1:\n        raise ValueError(\n            f\"Found multiple config files for {model_config.quantization}: \"\n            f\"{quant_config_files}\")\n\n    quant_config_file = quant_config_files[0]\n    with open(quant_config_file, \"r\") as f:\n        config = json.load(f)\n    return quant_cls.from_config(config)\n\n\ndef prepare_hf_model_weights(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    fall_back_to_pt: bool = True,\n    revision: Optional[str] = None,\n) -> Tuple[str, List[str], bool]:\n    # Download model weights from huggingface.\n    is_local = os.path.isdir(model_name_or_path)\n    use_safetensors = False\n    # Some quantized models use .pt files for storing the weights.\n    if load_format == \"auto\":\n        allow_patterns = [\"*.safetensors\", \"*.bin\"]\n    elif load_format == \"safetensors\":\n        use_safetensors = True\n        allow_patterns = [\"*.safetensors\"]\n    elif load_format == \"pt\":\n        allow_patterns = [\"*.pt\"]\n    elif load_format == \"npcache\":\n        allow_patterns = [\"*.bin\"]\n    else:\n        raise ValueError(f\"Unknown load_format: {load_format}\")\n\n    if fall_back_to_pt:\n        allow_patterns += [\"*.pt\"]\n\n    if not is_local:\n        # Before we download we look at that is available:\n        fs = HfFileSystem()\n        file_list = fs.ls(model_name_or_path, detail=False, revision=revision)\n\n        # depending on what is available we download different things\n        for pattern in allow_patterns:\n            matching = fnmatch.filter(file_list, pattern)\n            if len(matching) > 0:\n                allow_patterns = [pattern]\n                break\n\n        logger.info(f\"Using model weights format {allow_patterns}\")\n        # Use file lock to prevent multiple processes from\n        # downloading the same model weights at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          allow_patterns=allow_patterns,\n                                          cache_dir=cache_dir,\n                                          tqdm_class=Disabledtqdm,\n                                          revision=revision)\n    else:\n        hf_folder = model_name_or_path\n    hf_weights_files: List[str] = []\n    for pattern in allow_patterns:\n        hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))\n        if len(hf_weights_files) > 0:\n            if pattern == \"*.safetensors\":\n                use_safetensors = True\n            break\n    if not use_safetensors:\n        # Exclude files that are not needed for inference.\n        # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233\n        blacklist = [\n            \"training_args.bin\",\n            \"optimizer.bin\",\n            \"optimizer.pt\",\n            \"scheduler.pt\",\n            \"scaler.pt\",\n        ]\n        hf_weights_files = [\n            f for f in hf_weights_files\n            if not any(f.endswith(x) for x in blacklist)\n        ]\n\n    if len(hf_weights_files) == 0:\n        raise RuntimeError(\n            f\"Cannot find any model weights with `{model_name_or_path}`\")\n\n    return hf_folder, hf_weights_files, use_safetensors\n\n\ndef hf_model_weights_iterator(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    revision: Optional[str] = None,\n    fall_back_to_pt: Optional[bool] = True,\n) -> Iterator[Tuple[str, torch.Tensor]]:\n    hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(\n        model_name_or_path,\n        cache_dir=cache_dir,\n        load_format=load_format,\n        fall_back_to_pt=fall_back_to_pt,\n        revision=revision)\n\n    if load_format == \"npcache\":\n        # Currently np_cache only support *.bin checkpoints\n        assert use_safetensors is False\n\n        # Convert the model weights from torch tensors to numpy arrays for\n        # faster loading.\n        np_folder = os.path.join(hf_folder, \"np\")\n        os.makedirs(np_folder, exist_ok=True)\n        weight_names_file = os.path.join(np_folder, \"weight_names.json\")\n        # Use file lock to prevent multiple processes from\n        # dumping the same model weights to numpy at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            if not os.path.exists(weight_names_file):\n                weight_names = []\n                for bin_file in hf_weights_files:\n                    state = torch.load(bin_file, map_location=\"cpu\")\n                    for name, param in state.items():\n                        param_path = os.path.join(np_folder, name)\n                        with open(param_path, \"wb\") as f:\n                            np.save(f, param.cpu().detach().numpy())\n                        weight_names.append(name)\n                with open(weight_names_file, \"w\") as f:\n                    json.dump(weight_names, f)\n\n        with open(weight_names_file, \"r\") as f:\n            weight_names = json.load(f)\n\n        for name in weight_names:\n            param_path = os.path.join(np_folder, name)\n            with open(param_path, \"rb\") as f:\n                param = np.load(f)\n            yield name, torch.from_numpy(param)\n    elif use_safetensors:\n        for st_file in hf_weights_files:\n            with safe_open(st_file, framework=\"pt\") as f:\n                for name in f.keys():  # noqa: SIM118\n                    param = f.get_tensor(name)\n                    yield name, param\n    else:\n        for bin_file in hf_weights_files:\n            state = torch.load(bin_file, map_location=\"cpu\")\n            for name, param in state.items():\n                yield name, param\n            del state\n            torch.cuda.empty_cache()\n\n\ndef convert_pyslice_to_tensor(x: Any) -> torch.Tensor:\n    \"\"\"convert PySafeSlice object from safetensors to torch.Tensor\n\n    PySafeSlice object supports indexing, which is done before loading the\n    actual tensor and can reduce the amount of memory being read into the\n    memory. However, it does not support more advanced functionalities\n    like `.view()` or `.t()`. Therefore, if we need to modify the loaded\n    tensor with these more complicated operators, we need to convert to\n    tensor first.\n    \"\"\"\n    if not isinstance(x, torch.Tensor):\n        x = x[:]\n    return x\n\n\ndef default_weight_loader(param: torch.Tensor,\n                          loaded_weight: torch.Tensor) -> None:\n    \"\"\"Default weight loader.\"\"\"\n    assert param.size() == loaded_weight.size()\n    param.data.copy_(loaded_weight)\n\n\ndef initialize_dummy_weights(\n    model: torch.nn.Module,\n    low: float = -1e-3,\n    high: float = 1e-3,\n) -> None:\n    \"\"\"Initialize model weights with random values.\n\n    The model weights must be randomly initialized for accurate performance\n    measurements. Additionally, the model weights should not cause NaNs in the\n    forward pass. We empirically found that initializing the weights with\n    values between -1e-3 and 1e-3 works well for most models.\n    \"\"\"\n    for param in model.state_dict().values():\n        if torch.is_floating_point(param):\n            param.data.uniform_(low, high)\n", "path": "vllm/model_executor/weight_utils.py"}], "after_files": [{"content": "\"\"\"Utilities for downloading and initializing model weights.\"\"\"\nimport filelock\nimport glob\nimport fnmatch\nimport json\nimport os\nfrom collections import defaultdict\nfrom typing import Any, Iterator, List, Optional, Tuple\n\nfrom huggingface_hub import snapshot_download, HfFileSystem\nimport numpy as np\nfrom safetensors.torch import load_file, save_file, safe_open\nimport torch\nfrom tqdm.auto import tqdm\n\nfrom vllm.config import ModelConfig\nfrom vllm.logger import init_logger\nfrom vllm.model_executor.layers.quantization import (get_quantization_config,\n                                                     QuantizationConfig)\n\nlogger = init_logger(__name__)\n\n_xdg_cache_home = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))\n_vllm_filelocks_path = os.path.join(_xdg_cache_home, 'vllm/locks/')\n\n\nclass Disabledtqdm(tqdm):\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs, disable=True)\n\n\ndef get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):\n    lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path\n    os.makedirs(os.path.dirname(lock_dir), exist_ok=True)\n    lock_file_name = model_name_or_path.replace(\"/\", \"-\") + \".lock\"\n    lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))\n    return lock\n\n\ndef _shared_pointers(tensors):\n    ptrs = defaultdict(list)\n    for k, v in tensors.items():\n        ptrs[v.data_ptr()].append(k)\n    failing = []\n    for _, names in ptrs.items():\n        if len(names) > 1:\n            failing.append(names)\n    return failing\n\n\ndef convert_bin_to_safetensor_file(\n    pt_filename: str,\n    sf_filename: str,\n) -> None:\n    loaded = torch.load(pt_filename, map_location=\"cpu\")\n    if \"state_dict\" in loaded:\n        loaded = loaded[\"state_dict\"]\n    shared = _shared_pointers(loaded)\n    for shared_weights in shared:\n        for name in shared_weights[1:]:\n            loaded.pop(name)\n\n    # For tensors to be contiguous\n    loaded = {k: v.contiguous() for k, v in loaded.items()}\n\n    dirname = os.path.dirname(sf_filename)\n    os.makedirs(dirname, exist_ok=True)\n    save_file(loaded, sf_filename, metadata={\"format\": \"pt\"})\n\n    # check file size\n    sf_size = os.stat(sf_filename).st_size\n    pt_size = os.stat(pt_filename).st_size\n    if (sf_size - pt_size) / pt_size > 0.01:\n        raise RuntimeError(f\"\"\"The file size different is more than 1%:\n         - {sf_filename}: {sf_size}\n         - {pt_filename}: {pt_size}\n         \"\"\")\n\n    # check if the tensors are the same\n    reloaded = load_file(sf_filename)\n    for k in loaded:\n        pt_tensor = loaded[k]\n        sf_tensor = reloaded[k]\n        if not torch.equal(pt_tensor, sf_tensor):\n            raise RuntimeError(f\"The output tensors do not match for key {k}\")\n\n\n# TODO(woosuk): Move this to other place.\ndef get_quant_config(model_config: ModelConfig) -> QuantizationConfig:\n    quant_cls = get_quantization_config(model_config.quantization)\n    # Read the quantization config from the HF model config, if available.\n    hf_quant_config = getattr(model_config.hf_config, \"quantization_config\",\n                              None)\n    if hf_quant_config is not None:\n        return quant_cls.from_config(hf_quant_config)\n    model_name_or_path = model_config.model\n    is_local = os.path.isdir(model_name_or_path)\n    if not is_local:\n        # Download the config files.\n        with get_lock(model_name_or_path, model_config.download_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          revision=model_config.revision,\n                                          allow_patterns=\"*.json\",\n                                          cache_dir=model_config.download_dir,\n                                          tqdm_class=Disabledtqdm)\n    else:\n        hf_folder = model_name_or_path\n    config_files = glob.glob(os.path.join(hf_folder, \"*.json\"))\n\n    quant_config_files = [\n        f for f in config_files if any(\n            f.endswith(x) for x in quant_cls.get_config_filenames())\n    ]\n    if len(quant_config_files) == 0:\n        raise ValueError(\n            f\"Cannot find the config file for {model_config.quantization}\")\n    if len(quant_config_files) > 1:\n        raise ValueError(\n            f\"Found multiple config files for {model_config.quantization}: \"\n            f\"{quant_config_files}\")\n\n    quant_config_file = quant_config_files[0]\n    with open(quant_config_file, \"r\") as f:\n        config = json.load(f)\n    return quant_cls.from_config(config)\n\n\ndef prepare_hf_model_weights(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    fall_back_to_pt: bool = True,\n    revision: Optional[str] = None,\n) -> Tuple[str, List[str], bool]:\n    # Download model weights from huggingface.\n    is_local = os.path.isdir(model_name_or_path)\n    use_safetensors = False\n    # Some quantized models use .pt files for storing the weights.\n    if load_format == \"auto\":\n        allow_patterns = [\"*.safetensors\", \"*.bin\"]\n    elif load_format == \"safetensors\":\n        use_safetensors = True\n        allow_patterns = [\"*.safetensors\"]\n    elif load_format == \"pt\":\n        allow_patterns = [\"*.pt\"]\n    elif load_format == \"npcache\":\n        allow_patterns = [\"*.bin\"]\n    else:\n        raise ValueError(f\"Unknown load_format: {load_format}\")\n\n    if fall_back_to_pt:\n        allow_patterns += [\"*.pt\"]\n\n    if not is_local:\n        # Before we download we look at that is available:\n        fs = HfFileSystem()\n        file_list = fs.ls(model_name_or_path, detail=False, revision=revision)\n\n        # depending on what is available we download different things\n        for pattern in allow_patterns:\n            matching = fnmatch.filter(file_list, pattern)\n            if len(matching) > 0:\n                allow_patterns = [pattern]\n                break\n\n        logger.info(f\"Using model weights format {allow_patterns}\")\n        # Use file lock to prevent multiple processes from\n        # downloading the same model weights at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            hf_folder = snapshot_download(model_name_or_path,\n                                          allow_patterns=allow_patterns,\n                                          cache_dir=cache_dir,\n                                          tqdm_class=Disabledtqdm,\n                                          revision=revision)\n    else:\n        hf_folder = model_name_or_path\n    hf_weights_files: List[str] = []\n    for pattern in allow_patterns:\n        hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))\n        if len(hf_weights_files) > 0:\n            if pattern == \"*.safetensors\":\n                use_safetensors = True\n            break\n    if not use_safetensors:\n        # Exclude files that are not needed for inference.\n        # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233\n        blacklist = [\n            \"training_args.bin\",\n            \"optimizer.bin\",\n            \"optimizer.pt\",\n            \"scheduler.pt\",\n            \"scaler.pt\",\n        ]\n        hf_weights_files = [\n            f for f in hf_weights_files\n            if not any(f.endswith(x) for x in blacklist)\n        ]\n\n    if len(hf_weights_files) == 0:\n        raise RuntimeError(\n            f\"Cannot find any model weights with `{model_name_or_path}`\")\n\n    return hf_folder, hf_weights_files, use_safetensors\n\n\ndef hf_model_weights_iterator(\n    model_name_or_path: str,\n    cache_dir: Optional[str] = None,\n    load_format: str = \"auto\",\n    revision: Optional[str] = None,\n    fall_back_to_pt: Optional[bool] = True,\n) -> Iterator[Tuple[str, torch.Tensor]]:\n    hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(\n        model_name_or_path,\n        cache_dir=cache_dir,\n        load_format=load_format,\n        fall_back_to_pt=fall_back_to_pt,\n        revision=revision)\n\n    if load_format == \"npcache\":\n        # Currently np_cache only support *.bin checkpoints\n        assert use_safetensors is False\n\n        # Convert the model weights from torch tensors to numpy arrays for\n        # faster loading.\n        np_folder = os.path.join(hf_folder, \"np\")\n        os.makedirs(np_folder, exist_ok=True)\n        weight_names_file = os.path.join(np_folder, \"weight_names.json\")\n        # Use file lock to prevent multiple processes from\n        # dumping the same model weights to numpy at the same time.\n        with get_lock(model_name_or_path, cache_dir):\n            if not os.path.exists(weight_names_file):\n                weight_names = []\n                for bin_file in hf_weights_files:\n                    state = torch.load(bin_file, map_location=\"cpu\")\n                    for name, param in state.items():\n                        param_path = os.path.join(np_folder, name)\n                        with open(param_path, \"wb\") as f:\n                            np.save(f, param.cpu().detach().numpy())\n                        weight_names.append(name)\n                with open(weight_names_file, \"w\") as f:\n                    json.dump(weight_names, f)\n\n        with open(weight_names_file, \"r\") as f:\n            weight_names = json.load(f)\n\n        for name in weight_names:\n            param_path = os.path.join(np_folder, name)\n            with open(param_path, \"rb\") as f:\n                param = np.load(f)\n            yield name, torch.from_numpy(param)\n    elif use_safetensors:\n        for st_file in hf_weights_files:\n            with safe_open(st_file, framework=\"pt\") as f:\n                for name in f.keys():  # noqa: SIM118\n                    param = f.get_tensor(name)\n                    yield name, param\n    else:\n        for bin_file in hf_weights_files:\n            state = torch.load(bin_file, map_location=\"cpu\")\n            for name, param in state.items():\n                yield name, param\n            del state\n            torch.cuda.empty_cache()\n\n\ndef convert_pyslice_to_tensor(x: Any) -> torch.Tensor:\n    \"\"\"convert PySafeSlice object from safetensors to torch.Tensor\n\n    PySafeSlice object supports indexing, which is done before loading the\n    actual tensor and can reduce the amount of memory being read into the\n    memory. However, it does not support more advanced functionalities\n    like `.view()` or `.t()`. Therefore, if we need to modify the loaded\n    tensor with these more complicated operators, we need to convert to\n    tensor first.\n    \"\"\"\n    if not isinstance(x, torch.Tensor):\n        x = x[:]\n    return x\n\n\ndef default_weight_loader(param: torch.Tensor,\n                          loaded_weight: torch.Tensor) -> None:\n    \"\"\"Default weight loader.\"\"\"\n    assert param.size() == loaded_weight.size()\n    param.data.copy_(loaded_weight)\n\n\ndef initialize_dummy_weights(\n    model: torch.nn.Module,\n    low: float = -1e-3,\n    high: float = 1e-3,\n) -> None:\n    \"\"\"Initialize model weights with random values.\n\n    The model weights must be randomly initialized for accurate performance\n    measurements. Additionally, the model weights should not cause NaNs in the\n    forward pass. We empirically found that initializing the weights with\n    values between -1e-3 and 1e-3 works well for most models.\n    \"\"\"\n    for param in model.state_dict().values():\n        if torch.is_floating_point(param):\n            param.data.uniform_(low, high)\n", "path": "vllm/model_executor/weight_utils.py"}]}