We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[Minor] Fix test_cache.py CI test failure
While working on https://github.com/vllm-project/vllm/pull/2677, I noticed there was a test failure that came from a race condition from merging https://github.com/vllm-project/vllm/pull/2279 and https://github.com/vllm-project/vllm/pull/2616 "at the same time"

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/utils.py`
Content:
```
1 import enum
2 import os
3 import socket
4 import subprocess
5 import uuid
6 from platform import uname
7 from typing import List, Tuple, Union
8 from packaging.version import parse, Version
9 
10 import psutil
11 import torch
12 import asyncio
13 from functools import partial
14 from typing import (
15     Awaitable,
16     Callable,
17     TypeVar,
18 )
19 from collections import OrderedDict
20 from typing import Any, Hashable, Optional
21 
22 from vllm.logger import init_logger
23 
24 T = TypeVar("T")
25 logger = init_logger(__name__)
26 
27 STR_DTYPE_TO_TORCH_DTYPE = {
28     "half": torch.half,
29     "bfloat16": torch.bfloat16,
30     "float": torch.float,
31     "fp8_e5m2": torch.uint8,
32 }
33 
34 
35 class Device(enum.Enum):
36     GPU = enum.auto()
37     CPU = enum.auto()
38 
39 
40 class Counter:
41 
42     def __init__(self, start: int = 0) -> None:
43         self.counter = start
44 
45     def __next__(self) -> int:
46         i = self.counter
47         self.counter += 1
48         return i
49 
50     def reset(self) -> None:
51         self.counter = 0
52 
53 
54 class LRUCache:
55 
56     def __init__(self, capacity: int):
57         self.cache = OrderedDict()
58         self.capacity = capacity
59 
60     def __contains__(self, key: Hashable) -> bool:
61         return key in self.cache
62 
63     def __len__(self) -> int:
64         return len(self.cache)
65 
66     def __getitem__(self, key: Hashable) -> Any:
67         return self.get(key)
68 
69     def __setitem__(self, key: Hashable, value: Any) -> None:
70         self.put(key, value)
71 
72     def __delitem__(self, key: Hashable) -> None:
73         self.pop(key)
74 
75     def touch(self, key: Hashable) -> None:
76         self.cache.move_to_end(key)
77 
78     def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
79         if key in self.cache:
80             value = self.cache[key]
81             self.cache.move_to_end(key)
82         else:
83             value = default_value
84         return value
85 
86     def put(self, key: Hashable, value: Any) -> None:
87         self.cache[key] = value
88         self.cache.move_to_end(key)
89         self._remove_old_if_needed()
90 
91     def _on_remove(self, key: Hashable, value: Any):
92         pass
93 
94     def remove_oldest(self):
95         if not self.cache:
96             return
97         key, value = self.cache.popitem(last=False)
98         self._on_remove(key, value)
99 
100     def _remove_old_if_needed(self) -> None:
101         while len(self.cache) > self.capacity:
102             self.remove_oldest()
103 
104     def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
105         run_on_remove = key in self.cache
106         value = self.cache.pop(key, default_value)
107         if run_on_remove:
108             self._on_remove(key, value)
109         return value
110 
111     def clear(self):
112         while len(self.cache) > 0:
113             self.remove_oldest()
114         self.cache.clear()
115 
116 
117 def is_hip() -> bool:
118     return torch.version.hip is not None
119 
120 
121 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
122     """Returns the maximum shared memory per thread block in bytes."""
123     # NOTE: This import statement should be executed lazily since
124     # the Neuron-X backend does not have the `cuda_utils` module.
125     from vllm._C import cuda_utils
126 
127     max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(
128         gpu)
129     # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail
130     assert max_shared_mem > 0, "max_shared_mem can not be zero"
131     return int(max_shared_mem)
132 
133 
134 def get_cpu_memory() -> int:
135     """Returns the total CPU memory of the node in bytes."""
136     return psutil.virtual_memory().total
137 
138 
139 def random_uuid() -> str:
140     return str(uuid.uuid4().hex)
141 
142 
143 def in_wsl() -> bool:
144     # Reference: https://github.com/microsoft/WSL/issues/4071
145     return "microsoft" in " ".join(uname()).lower()
146 
147 
148 def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
149     """Take a blocking function, and run it on in an executor thread.
150 
151     This function prevents the blocking function from blocking the
152     asyncio event loop.
153     The code in this function needs to be thread safe.
154     """
155 
156     def _async_wrapper(*args, **kwargs) -> asyncio.Future:
157         loop = asyncio.get_event_loop()
158         p_func = partial(func, *args, **kwargs)
159         return loop.run_in_executor(executor=None, func=p_func)
160 
161     return _async_wrapper
162 
163 
164 def get_ip() -> str:
165     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
166     s.connect(("8.8.8.8", 80))  # Doesn't need to be reachable
167     return s.getsockname()[0]
168 
169 
170 def get_distributed_init_method(ip: str, port: int) -> str:
171     return f"tcp://{ip}:{port}"
172 
173 
174 def get_open_port() -> int:
175     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
176         s.bind(("", 0))
177         return s.getsockname()[1]
178 
179 
180 def set_cuda_visible_devices(device_ids: List[int]) -> None:
181     os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
182 
183 
184 def get_nvcc_cuda_version() -> Version:
185     cuda_home = os.environ.get('CUDA_HOME')
186     if not cuda_home:
187         cuda_home = '/usr/local/cuda'
188         logger.info(
189             f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
190         )
191     nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
192                                           universal_newlines=True)
193     output = nvcc_output.split()
194     release_idx = output.index("release") + 1
195     nvcc_cuda_version = parse(output[release_idx].split(",")[0])
196     return nvcc_cuda_version
197 
198 
199 def _generate_random_fp8_e5m2(
200     tensor: torch.tensor,
201     low: float,
202     high: float,
203 ) -> None:
204     # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,
205     # it may occur Inf or NaN if we directly use torch.randint
206     # to generate random data for fp8 data.
207     # For example, s.11111.00 in fp8e5m2 format repesents Inf.
208     #     | E4M3        | E5M2
209     #-----|-------------|-------------------
210     # Inf | N/A         | s.11111.00
211     # NaN | s.1111.111  | s.11111.{01,10,11}
212     from vllm._C import cache_ops
213     tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
214     tensor_tmp.uniform_(low, high)
215     cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
216     del tensor_tmp
217 
218 
219 def create_kv_caches_with_random(
220     num_blocks: int,
221     block_size: int,
222     num_layers: int,
223     num_heads: int,
224     head_size: int,
225     cache_dtype: Optional[Union[str, torch.dtype]],
226     model_dtype: Optional[Union[str, torch.dtype]] = None,
227     seed: Optional[int] = 0,
228     device: Optional[str] = "cuda",
229 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
230     torch.random.manual_seed(seed)
231     if torch.cuda.is_available():
232         torch.cuda.manual_seed(seed)
233 
234     if isinstance(cache_dtype, str):
235         if cache_dtype == "auto":
236             if isinstance(model_dtype, str):
237                 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
238             elif isinstance(model_dtype, torch.dtype):
239                 torch_dtype = model_dtype
240             else:
241                 raise ValueError(f"Invalid model dtype: {model_dtype}")
242         elif cache_dtype in ["half", "bfloat16", "float"]:
243             torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
244         elif cache_dtype == "fp8_e5m2":
245             torch_dtype = torch.uint8
246         else:
247             raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
248     elif isinstance(cache_dtype, torch.dtype):
249         torch_dtype = cache_dtype
250     else:
251         raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
252 
253     scale = head_size**-0.5
254     x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
255     key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
256     key_caches = []
257     for _ in range(num_layers):
258         key_cache = torch.empty(size=key_cache_shape,
259                                 dtype=torch_dtype,
260                                 device=device)
261         if cache_dtype in ["auto", "half", "bfloat16", "float"]:
262             key_cache.uniform_(-scale, scale)
263         elif cache_dtype == 'fp8_e5m2':
264             _generate_random_fp8_e5m2(key_cache, -scale, scale)
265         key_caches.append(key_cache)
266 
267     value_cache_shape = (num_blocks, num_heads, head_size, block_size)
268     value_caches = []
269     for _ in range(num_layers):
270         value_cache = torch.empty(size=value_cache_shape,
271                                   dtype=torch_dtype,
272                                   device=device)
273         if cache_dtype in ["auto", "half", "bfloat16", "float"]:
274             value_cache.uniform_(-scale, scale)
275         elif cache_dtype == 'fp8_e5m2':
276             _generate_random_fp8_e5m2(value_cache, -scale, scale)
277         value_caches.append(value_cache)
278     return key_caches, value_caches
279
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/utils.py b/vllm/utils.py
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -258,10 +258,13 @@
         key_cache = torch.empty(size=key_cache_shape,
                                 dtype=torch_dtype,
                                 device=device)
-        if cache_dtype in ["auto", "half", "bfloat16", "float"]:
-            key_cache.uniform_(-scale, scale)
-        elif cache_dtype == 'fp8_e5m2':
+        if cache_dtype == 'fp8_e5m2':
             _generate_random_fp8_e5m2(key_cache, -scale, scale)
+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
+            key_cache.uniform_(-scale, scale)
+        else:
+            raise ValueError(
+                f"Does not support key cache of type {cache_dtype}")
         key_caches.append(key_cache)
 
     value_cache_shape = (num_blocks, num_heads, head_size, block_size)
@@ -270,9 +273,12 @@
         value_cache = torch.empty(size=value_cache_shape,
                                   dtype=torch_dtype,
                                   device=device)
-        if cache_dtype in ["auto", "half", "bfloat16", "float"]:
-            value_cache.uniform_(-scale, scale)
-        elif cache_dtype == 'fp8_e5m2':
+        if cache_dtype == 'fp8_e5m2':
             _generate_random_fp8_e5m2(value_cache, -scale, scale)
+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
+            value_cache.uniform_(-scale, scale)
+        else:
+            raise ValueError(
+                f"Does not support value cache of type {cache_dtype}")
         value_caches.append(value_cache)
     return key_caches, value_caches

{"golden_diff": "diff --git a/vllm/utils.py b/vllm/utils.py\n--- a/vllm/utils.py\n+++ b/vllm/utils.py\n@@ -258,10 +258,13 @@\n         key_cache = torch.empty(size=key_cache_shape,\n                                 dtype=torch_dtype,\n                                 device=device)\n-        if cache_dtype in [\"auto\", \"half\", \"bfloat16\", \"float\"]:\n-            key_cache.uniform_(-scale, scale)\n-        elif cache_dtype == 'fp8_e5m2':\n+        if cache_dtype == 'fp8_e5m2':\n             _generate_random_fp8_e5m2(key_cache, -scale, scale)\n+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n+            key_cache.uniform_(-scale, scale)\n+        else:\n+            raise ValueError(\n+                f\"Does not support key cache of type {cache_dtype}\")\n         key_caches.append(key_cache)\n \n     value_cache_shape = (num_blocks, num_heads, head_size, block_size)\n@@ -270,9 +273,12 @@\n         value_cache = torch.empty(size=value_cache_shape,\n                                   dtype=torch_dtype,\n                                   device=device)\n-        if cache_dtype in [\"auto\", \"half\", \"bfloat16\", \"float\"]:\n-            value_cache.uniform_(-scale, scale)\n-        elif cache_dtype == 'fp8_e5m2':\n+        if cache_dtype == 'fp8_e5m2':\n             _generate_random_fp8_e5m2(value_cache, -scale, scale)\n+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n+            value_cache.uniform_(-scale, scale)\n+        else:\n+            raise ValueError(\n+                f\"Does not support value cache of type {cache_dtype}\")\n         value_caches.append(value_cache)\n     return key_caches, value_caches\n", "issue": "[Minor] Fix test_cache.py CI test failure\nWhile working on https://github.com/vllm-project/vllm/pull/2677, I noticed there was a test failure that came from a race condition from merging https://github.com/vllm-project/vllm/pull/2279 and https://github.com/vllm-project/vllm/pull/2616 \"at the same time\"\n", "before_files": [{"content": "import enum\nimport os\nimport socket\nimport subprocess\nimport uuid\nfrom platform import uname\nfrom typing import List, Tuple, Union\nfrom packaging.version import parse, Version\n\nimport psutil\nimport torch\nimport asyncio\nfrom functools import partial\nfrom typing import (\n    Awaitable,\n    Callable,\n    TypeVar,\n)\nfrom collections import OrderedDict\nfrom typing import Any, Hashable, Optional\n\nfrom vllm.logger import init_logger\n\nT = TypeVar(\"T\")\nlogger = init_logger(__name__)\n\nSTR_DTYPE_TO_TORCH_DTYPE = {\n    \"half\": torch.half,\n    \"bfloat16\": torch.bfloat16,\n    \"float\": torch.float,\n    \"fp8_e5m2\": torch.uint8,\n}\n\n\nclass Device(enum.Enum):\n    GPU = enum.auto()\n    CPU = enum.auto()\n\n\nclass Counter:\n\n    def __init__(self, start: int = 0) -> None:\n        self.counter = start\n\n    def __next__(self) -> int:\n        i = self.counter\n        self.counter += 1\n        return i\n\n    def reset(self) -> None:\n        self.counter = 0\n\n\nclass LRUCache:\n\n    def __init__(self, capacity: int):\n        self.cache = OrderedDict()\n        self.capacity = capacity\n\n    def __contains__(self, key: Hashable) -> bool:\n        return key in self.cache\n\n    def __len__(self) -> int:\n        return len(self.cache)\n\n    def __getitem__(self, key: Hashable) -> Any:\n        return self.get(key)\n\n    def __setitem__(self, key: Hashable, value: Any) -> None:\n        self.put(key, value)\n\n    def __delitem__(self, key: Hashable) -> None:\n        self.pop(key)\n\n    def touch(self, key: Hashable) -> None:\n        self.cache.move_to_end(key)\n\n    def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:\n        if key in self.cache:\n            value = self.cache[key]\n            self.cache.move_to_end(key)\n        else:\n            value = default_value\n        return value\n\n    def put(self, key: Hashable, value: Any) -> None:\n        self.cache[key] = value\n        self.cache.move_to_end(key)\n        self._remove_old_if_needed()\n\n    def _on_remove(self, key: Hashable, value: Any):\n        pass\n\n    def remove_oldest(self):\n        if not self.cache:\n            return\n        key, value = self.cache.popitem(last=False)\n        self._on_remove(key, value)\n\n    def _remove_old_if_needed(self) -> None:\n        while len(self.cache) > self.capacity:\n            self.remove_oldest()\n\n    def pop(self, key: int, default_value: Optional[Any] = None) -> Any:\n        run_on_remove = key in self.cache\n        value = self.cache.pop(key, default_value)\n        if run_on_remove:\n            self._on_remove(key, value)\n        return value\n\n    def clear(self):\n        while len(self.cache) > 0:\n            self.remove_oldest()\n        self.cache.clear()\n\n\ndef is_hip() -> bool:\n    return torch.version.hip is not None\n\n\ndef get_max_shared_memory_bytes(gpu: int = 0) -> int:\n    \"\"\"Returns the maximum shared memory per thread block in bytes.\"\"\"\n    # NOTE: This import statement should be executed lazily since\n    # the Neuron-X backend does not have the `cuda_utils` module.\n    from vllm._C import cuda_utils\n\n    max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(\n        gpu)\n    # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail\n    assert max_shared_mem > 0, \"max_shared_mem can not be zero\"\n    return int(max_shared_mem)\n\n\ndef get_cpu_memory() -> int:\n    \"\"\"Returns the total CPU memory of the node in bytes.\"\"\"\n    return psutil.virtual_memory().total\n\n\ndef random_uuid() -> str:\n    return str(uuid.uuid4().hex)\n\n\ndef in_wsl() -> bool:\n    # Reference: https://github.com/microsoft/WSL/issues/4071\n    return \"microsoft\" in \" \".join(uname()).lower()\n\n\ndef make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:\n    \"\"\"Take a blocking function, and run it on in an executor thread.\n\n    This function prevents the blocking function from blocking the\n    asyncio event loop.\n    The code in this function needs to be thread safe.\n    \"\"\"\n\n    def _async_wrapper(*args, **kwargs) -> asyncio.Future:\n        loop = asyncio.get_event_loop()\n        p_func = partial(func, *args, **kwargs)\n        return loop.run_in_executor(executor=None, func=p_func)\n\n    return _async_wrapper\n\n\ndef get_ip() -> str:\n    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n    s.connect((\"8.8.8.8\", 80))  # Doesn't need to be reachable\n    return s.getsockname()[0]\n\n\ndef get_distributed_init_method(ip: str, port: int) -> str:\n    return f\"tcp://{ip}:{port}\"\n\n\ndef get_open_port() -> int:\n    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n        s.bind((\"\", 0))\n        return s.getsockname()[1]\n\n\ndef set_cuda_visible_devices(device_ids: List[int]) -> None:\n    os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, device_ids))\n\n\ndef get_nvcc_cuda_version() -> Version:\n    cuda_home = os.environ.get('CUDA_HOME')\n    if not cuda_home:\n        cuda_home = '/usr/local/cuda'\n        logger.info(\n            f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'\n        )\n    nvcc_output = subprocess.check_output([cuda_home + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef _generate_random_fp8_e5m2(\n    tensor: torch.tensor,\n    low: float,\n    high: float,\n) -> None:\n    # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,\n    # it may occur Inf or NaN if we directly use torch.randint\n    # to generate random data for fp8 data.\n    # For example, s.11111.00 in fp8e5m2 format repesents Inf.\n    #     | E4M3        | E5M2\n    #-----|-------------|-------------------\n    # Inf | N/A         | s.11111.00\n    # NaN | s.1111.111  | s.11111.{01,10,11}\n    from vllm._C import cache_ops\n    tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)\n    tensor_tmp.uniform_(low, high)\n    cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)\n    del tensor_tmp\n\n\ndef create_kv_caches_with_random(\n    num_blocks: int,\n    block_size: int,\n    num_layers: int,\n    num_heads: int,\n    head_size: int,\n    cache_dtype: Optional[Union[str, torch.dtype]],\n    model_dtype: Optional[Union[str, torch.dtype]] = None,\n    seed: Optional[int] = 0,\n    device: Optional[str] = \"cuda\",\n) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:\n    torch.random.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n\n    if isinstance(cache_dtype, str):\n        if cache_dtype == \"auto\":\n            if isinstance(model_dtype, str):\n                torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]\n            elif isinstance(model_dtype, torch.dtype):\n                torch_dtype = model_dtype\n            else:\n                raise ValueError(f\"Invalid model dtype: {model_dtype}\")\n        elif cache_dtype in [\"half\", \"bfloat16\", \"float\"]:\n            torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]\n        elif cache_dtype == \"fp8_e5m2\":\n            torch_dtype = torch.uint8\n        else:\n            raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n    elif isinstance(cache_dtype, torch.dtype):\n        torch_dtype = cache_dtype\n    else:\n        raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n\n    scale = head_size**-0.5\n    x = 16 // torch.tensor([], dtype=torch_dtype).element_size()\n    key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)\n    key_caches = []\n    for _ in range(num_layers):\n        key_cache = torch.empty(size=key_cache_shape,\n                                dtype=torch_dtype,\n                                device=device)\n        if cache_dtype in [\"auto\", \"half\", \"bfloat16\", \"float\"]:\n            key_cache.uniform_(-scale, scale)\n        elif cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(key_cache, -scale, scale)\n        key_caches.append(key_cache)\n\n    value_cache_shape = (num_blocks, num_heads, head_size, block_size)\n    value_caches = []\n    for _ in range(num_layers):\n        value_cache = torch.empty(size=value_cache_shape,\n                                  dtype=torch_dtype,\n                                  device=device)\n        if cache_dtype in [\"auto\", \"half\", \"bfloat16\", \"float\"]:\n            value_cache.uniform_(-scale, scale)\n        elif cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(value_cache, -scale, scale)\n        value_caches.append(value_cache)\n    return key_caches, value_caches\n", "path": "vllm/utils.py"}], "after_files": [{"content": "import enum\nimport os\nimport socket\nimport subprocess\nimport uuid\nfrom platform import uname\nfrom typing import List, Tuple, Union\nfrom packaging.version import parse, Version\n\nimport psutil\nimport torch\nimport asyncio\nfrom functools import partial\nfrom typing import (\n    Awaitable,\n    Callable,\n    TypeVar,\n)\nfrom collections import OrderedDict\nfrom typing import Any, Hashable, Optional\n\nfrom vllm.logger import init_logger\n\nT = TypeVar(\"T\")\nlogger = init_logger(__name__)\n\nSTR_DTYPE_TO_TORCH_DTYPE = {\n    \"half\": torch.half,\n    \"bfloat16\": torch.bfloat16,\n    \"float\": torch.float,\n    \"fp8_e5m2\": torch.uint8,\n}\n\n\nclass Device(enum.Enum):\n    GPU = enum.auto()\n    CPU = enum.auto()\n\n\nclass Counter:\n\n    def __init__(self, start: int = 0) -> None:\n        self.counter = start\n\n    def __next__(self) -> int:\n        i = self.counter\n        self.counter += 1\n        return i\n\n    def reset(self) -> None:\n        self.counter = 0\n\n\nclass LRUCache:\n\n    def __init__(self, capacity: int):\n        self.cache = OrderedDict()\n        self.capacity = capacity\n\n    def __contains__(self, key: Hashable) -> bool:\n        return key in self.cache\n\n    def __len__(self) -> int:\n        return len(self.cache)\n\n    def __getitem__(self, key: Hashable) -> Any:\n        return self.get(key)\n\n    def __setitem__(self, key: Hashable, value: Any) -> None:\n        self.put(key, value)\n\n    def __delitem__(self, key: Hashable) -> None:\n        self.pop(key)\n\n    def touch(self, key: Hashable) -> None:\n        self.cache.move_to_end(key)\n\n    def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:\n        if key in self.cache:\n            value = self.cache[key]\n            self.cache.move_to_end(key)\n        else:\n            value = default_value\n        return value\n\n    def put(self, key: Hashable, value: Any) -> None:\n        self.cache[key] = value\n        self.cache.move_to_end(key)\n        self._remove_old_if_needed()\n\n    def _on_remove(self, key: Hashable, value: Any):\n        pass\n\n    def remove_oldest(self):\n        if not self.cache:\n            return\n        key, value = self.cache.popitem(last=False)\n        self._on_remove(key, value)\n\n    def _remove_old_if_needed(self) -> None:\n        while len(self.cache) > self.capacity:\n            self.remove_oldest()\n\n    def pop(self, key: int, default_value: Optional[Any] = None) -> Any:\n        run_on_remove = key in self.cache\n        value = self.cache.pop(key, default_value)\n        if run_on_remove:\n            self._on_remove(key, value)\n        return value\n\n    def clear(self):\n        while len(self.cache) > 0:\n            self.remove_oldest()\n        self.cache.clear()\n\n\ndef is_hip() -> bool:\n    return torch.version.hip is not None\n\n\ndef get_max_shared_memory_bytes(gpu: int = 0) -> int:\n    \"\"\"Returns the maximum shared memory per thread block in bytes.\"\"\"\n    # NOTE: This import statement should be executed lazily since\n    # the Neuron-X backend does not have the `cuda_utils` module.\n    from vllm._C import cuda_utils\n\n    max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(\n        gpu)\n    # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail\n    assert max_shared_mem > 0, \"max_shared_mem can not be zero\"\n    return int(max_shared_mem)\n\n\ndef get_cpu_memory() -> int:\n    \"\"\"Returns the total CPU memory of the node in bytes.\"\"\"\n    return psutil.virtual_memory().total\n\n\ndef random_uuid() -> str:\n    return str(uuid.uuid4().hex)\n\n\ndef in_wsl() -> bool:\n    # Reference: https://github.com/microsoft/WSL/issues/4071\n    return \"microsoft\" in \" \".join(uname()).lower()\n\n\ndef make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:\n    \"\"\"Take a blocking function, and run it on in an executor thread.\n\n    This function prevents the blocking function from blocking the\n    asyncio event loop.\n    The code in this function needs to be thread safe.\n    \"\"\"\n\n    def _async_wrapper(*args, **kwargs) -> asyncio.Future:\n        loop = asyncio.get_event_loop()\n        p_func = partial(func, *args, **kwargs)\n        return loop.run_in_executor(executor=None, func=p_func)\n\n    return _async_wrapper\n\n\ndef get_ip() -> str:\n    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n    s.connect((\"8.8.8.8\", 80))  # Doesn't need to be reachable\n    return s.getsockname()[0]\n\n\ndef get_distributed_init_method(ip: str, port: int) -> str:\n    return f\"tcp://{ip}:{port}\"\n\n\ndef get_open_port() -> int:\n    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n        s.bind((\"\", 0))\n        return s.getsockname()[1]\n\n\ndef set_cuda_visible_devices(device_ids: List[int]) -> None:\n    os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, device_ids))\n\n\ndef get_nvcc_cuda_version() -> Version:\n    cuda_home = os.environ.get('CUDA_HOME')\n    if not cuda_home:\n        cuda_home = '/usr/local/cuda'\n        logger.info(\n            f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'\n        )\n    nvcc_output = subprocess.check_output([cuda_home + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef _generate_random_fp8_e5m2(\n    tensor: torch.tensor,\n    low: float,\n    high: float,\n) -> None:\n    # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,\n    # it may occur Inf or NaN if we directly use torch.randint\n    # to generate random data for fp8 data.\n    # For example, s.11111.00 in fp8e5m2 format repesents Inf.\n    #     | E4M3        | E5M2\n    #-----|-------------|-------------------\n    # Inf | N/A         | s.11111.00\n    # NaN | s.1111.111  | s.11111.{01,10,11}\n    from vllm._C import cache_ops\n    tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)\n    tensor_tmp.uniform_(low, high)\n    cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)\n    del tensor_tmp\n\n\ndef create_kv_caches_with_random(\n    num_blocks: int,\n    block_size: int,\n    num_layers: int,\n    num_heads: int,\n    head_size: int,\n    cache_dtype: Optional[Union[str, torch.dtype]],\n    model_dtype: Optional[Union[str, torch.dtype]] = None,\n    seed: Optional[int] = 0,\n    device: Optional[str] = \"cuda\",\n) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:\n    torch.random.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n\n    if isinstance(cache_dtype, str):\n        if cache_dtype == \"auto\":\n            if isinstance(model_dtype, str):\n                torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]\n            elif isinstance(model_dtype, torch.dtype):\n                torch_dtype = model_dtype\n            else:\n                raise ValueError(f\"Invalid model dtype: {model_dtype}\")\n        elif cache_dtype in [\"half\", \"bfloat16\", \"float\"]:\n            torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]\n        elif cache_dtype == \"fp8_e5m2\":\n            torch_dtype = torch.uint8\n        else:\n            raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n    elif isinstance(cache_dtype, torch.dtype):\n        torch_dtype = cache_dtype\n    else:\n        raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n\n    scale = head_size**-0.5\n    x = 16 // torch.tensor([], dtype=torch_dtype).element_size()\n    key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)\n    key_caches = []\n    for _ in range(num_layers):\n        key_cache = torch.empty(size=key_cache_shape,\n                                dtype=torch_dtype,\n                                device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(key_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            key_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support key cache of type {cache_dtype}\")\n        key_caches.append(key_cache)\n\n    value_cache_shape = (num_blocks, num_heads, head_size, block_size)\n    value_caches = []\n    for _ in range(num_layers):\n        value_cache = torch.empty(size=value_cache_shape,\n                                  dtype=torch_dtype,\n                                  device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(value_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            value_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support value cache of type {cache_dtype}\")\n        value_caches.append(value_cache)\n    return key_caches, value_caches\n", "path": "vllm/utils.py"}]}