We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
why vllm==0.3.3 need to access google
![微信图片_20240304112403](https://github.com/vllm-project/vllm/assets/38678334/f21e1ec0-bd6f-4b26-aeee-d6e4e5822fc2)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `vllm/utils.py`
Content:
```
1 import enum
2 import os
3 import socket
4 import subprocess
5 import uuid
6 from platform import uname
7 from typing import List, Tuple, Union
8 from packaging.version import parse, Version
9 
10 import psutil
11 import torch
12 import asyncio
13 from functools import partial
14 from typing import (
15     Awaitable,
16     Callable,
17     TypeVar,
18 )
19 from collections import OrderedDict
20 from typing import Any, Hashable, Optional
21 
22 from vllm.logger import init_logger
23 
24 T = TypeVar("T")
25 logger = init_logger(__name__)
26 
27 STR_DTYPE_TO_TORCH_DTYPE = {
28     "half": torch.half,
29     "bfloat16": torch.bfloat16,
30     "float": torch.float,
31     "fp8_e5m2": torch.uint8,
32 }
33 
34 
35 class Device(enum.Enum):
36     GPU = enum.auto()
37     CPU = enum.auto()
38 
39 
40 class Counter:
41 
42     def __init__(self, start: int = 0) -> None:
43         self.counter = start
44 
45     def __next__(self) -> int:
46         i = self.counter
47         self.counter += 1
48         return i
49 
50     def reset(self) -> None:
51         self.counter = 0
52 
53 
54 class LRUCache:
55 
56     def __init__(self, capacity: int):
57         self.cache = OrderedDict()
58         self.capacity = capacity
59 
60     def __contains__(self, key: Hashable) -> bool:
61         return key in self.cache
62 
63     def __len__(self) -> int:
64         return len(self.cache)
65 
66     def __getitem__(self, key: Hashable) -> Any:
67         return self.get(key)
68 
69     def __setitem__(self, key: Hashable, value: Any) -> None:
70         self.put(key, value)
71 
72     def __delitem__(self, key: Hashable) -> None:
73         self.pop(key)
74 
75     def touch(self, key: Hashable) -> None:
76         self.cache.move_to_end(key)
77 
78     def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
79         if key in self.cache:
80             value = self.cache[key]
81             self.cache.move_to_end(key)
82         else:
83             value = default_value
84         return value
85 
86     def put(self, key: Hashable, value: Any) -> None:
87         self.cache[key] = value
88         self.cache.move_to_end(key)
89         self._remove_old_if_needed()
90 
91     def _on_remove(self, key: Hashable, value: Any):
92         pass
93 
94     def remove_oldest(self):
95         if not self.cache:
96             return
97         key, value = self.cache.popitem(last=False)
98         self._on_remove(key, value)
99 
100     def _remove_old_if_needed(self) -> None:
101         while len(self.cache) > self.capacity:
102             self.remove_oldest()
103 
104     def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
105         run_on_remove = key in self.cache
106         value = self.cache.pop(key, default_value)
107         if run_on_remove:
108             self._on_remove(key, value)
109         return value
110 
111     def clear(self):
112         while len(self.cache) > 0:
113             self.remove_oldest()
114         self.cache.clear()
115 
116 
117 def is_hip() -> bool:
118     return torch.version.hip is not None
119 
120 
121 def is_neuron() -> bool:
122     try:
123         import transformers_neuronx
124     except ImportError:
125         transformers_neuronx = None
126     return transformers_neuronx is not None
127 
128 
129 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
130     """Returns the maximum shared memory per thread block in bytes."""
131     # NOTE: This import statement should be executed lazily since
132     # the Neuron-X backend does not have the `cuda_utils` module.
133     from vllm._C import cuda_utils
134 
135     max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(
136         gpu)
137     # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail
138     assert max_shared_mem > 0, "max_shared_mem can not be zero"
139     return int(max_shared_mem)
140 
141 
142 def get_cpu_memory() -> int:
143     """Returns the total CPU memory of the node in bytes."""
144     return psutil.virtual_memory().total
145 
146 
147 def random_uuid() -> str:
148     return str(uuid.uuid4().hex)
149 
150 
151 def in_wsl() -> bool:
152     # Reference: https://github.com/microsoft/WSL/issues/4071
153     return "microsoft" in " ".join(uname()).lower()
154 
155 
156 def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
157     """Take a blocking function, and run it on in an executor thread.
158 
159     This function prevents the blocking function from blocking the
160     asyncio event loop.
161     The code in this function needs to be thread safe.
162     """
163 
164     def _async_wrapper(*args, **kwargs) -> asyncio.Future:
165         loop = asyncio.get_event_loop()
166         p_func = partial(func, *args, **kwargs)
167         return loop.run_in_executor(executor=None, func=p_func)
168 
169     return _async_wrapper
170 
171 
172 def get_ip() -> str:
173     # try ipv4
174     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
175     try:
176         s.connect(("dns.google", 80))  # Doesn't need to be reachable
177         return s.getsockname()[0]
178     except OSError:
179         # try ipv6
180         s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
181         s.connect(("dns.google", 80))
182         return s.getsockname()[0]
183 
184 
185 def get_distributed_init_method(ip: str, port: int) -> str:
186     return f"tcp://{ip}:{port}"
187 
188 
189 def get_open_port() -> int:
190     # try ipv4
191     try:
192         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
193             s.bind(("", 0))
194             return s.getsockname()[1]
195     except OSError:
196         # try ipv6
197         with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s:
198             s.bind(("", 0))
199             return s.getsockname()[1]
200 
201 
202 def set_cuda_visible_devices(device_ids: List[int]) -> None:
203     os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
204 
205 
206 def get_nvcc_cuda_version() -> Optional[Version]:
207     cuda_home = os.environ.get('CUDA_HOME')
208     if not cuda_home:
209         cuda_home = '/usr/local/cuda'
210         if os.path.isfile(cuda_home + '/bin/nvcc'):
211             logger.info(
212                 f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
213             )
214         else:
215             logger.warning(
216                 f'Not found nvcc in {cuda_home}. Skip cuda version check!')
217             return None
218     nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
219                                           universal_newlines=True)
220     output = nvcc_output.split()
221     release_idx = output.index("release") + 1
222     nvcc_cuda_version = parse(output[release_idx].split(",")[0])
223     return nvcc_cuda_version
224 
225 
226 def _generate_random_fp8_e5m2(
227     tensor: torch.tensor,
228     low: float,
229     high: float,
230 ) -> None:
231     # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,
232     # it may occur Inf or NaN if we directly use torch.randint
233     # to generate random data for fp8 data.
234     # For example, s.11111.00 in fp8e5m2 format represents Inf.
235     #     | E4M3        | E5M2
236     #-----|-------------|-------------------
237     # Inf | N/A         | s.11111.00
238     # NaN | s.1111.111  | s.11111.{01,10,11}
239     from vllm._C import cache_ops
240     tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
241     tensor_tmp.uniform_(low, high)
242     cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
243     del tensor_tmp
244 
245 
246 def create_kv_caches_with_random(
247     num_blocks: int,
248     block_size: int,
249     num_layers: int,
250     num_heads: int,
251     head_size: int,
252     cache_dtype: Optional[Union[str, torch.dtype]],
253     model_dtype: Optional[Union[str, torch.dtype]] = None,
254     seed: Optional[int] = 0,
255     device: Optional[str] = "cuda",
256 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
257     torch.random.manual_seed(seed)
258     if torch.cuda.is_available():
259         torch.cuda.manual_seed(seed)
260 
261     if isinstance(cache_dtype, str):
262         if cache_dtype == "auto":
263             if isinstance(model_dtype, str):
264                 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
265             elif isinstance(model_dtype, torch.dtype):
266                 torch_dtype = model_dtype
267             else:
268                 raise ValueError(f"Invalid model dtype: {model_dtype}")
269         elif cache_dtype in ["half", "bfloat16", "float"]:
270             torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
271         elif cache_dtype == "fp8_e5m2":
272             torch_dtype = torch.uint8
273         else:
274             raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
275     elif isinstance(cache_dtype, torch.dtype):
276         torch_dtype = cache_dtype
277     else:
278         raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
279 
280     scale = head_size**-0.5
281     x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
282     key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
283     key_caches = []
284     for _ in range(num_layers):
285         key_cache = torch.empty(size=key_cache_shape,
286                                 dtype=torch_dtype,
287                                 device=device)
288         if cache_dtype == 'fp8_e5m2':
289             _generate_random_fp8_e5m2(key_cache, -scale, scale)
290         elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
291             key_cache.uniform_(-scale, scale)
292         else:
293             raise ValueError(
294                 f"Does not support key cache of type {cache_dtype}")
295         key_caches.append(key_cache)
296 
297     value_cache_shape = (num_blocks, num_heads, head_size, block_size)
298     value_caches = []
299     for _ in range(num_layers):
300         value_cache = torch.empty(size=value_cache_shape,
301                                   dtype=torch_dtype,
302                                   device=device)
303         if cache_dtype == 'fp8_e5m2':
304             _generate_random_fp8_e5m2(value_cache, -scale, scale)
305         elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
306             value_cache.uniform_(-scale, scale)
307         else:
308             raise ValueError(
309                 f"Does not support value cache of type {cache_dtype}")
310         value_caches.append(value_cache)
311     return key_caches, value_caches
312
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/vllm/utils.py b/vllm/utils.py
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -173,7 +173,7 @@
     # try ipv4
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
-        s.connect(("dns.google", 80))  # Doesn't need to be reachable
+        s.connect(("8.8.8.8", 80))  # Doesn't need to be reachable
         return s.getsockname()[0]
     except OSError:
         # try ipv6

{"golden_diff": "diff --git a/vllm/utils.py b/vllm/utils.py\n--- a/vllm/utils.py\n+++ b/vllm/utils.py\n@@ -173,7 +173,7 @@\n     # try ipv4\n     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n     try:\n-        s.connect((\"dns.google\", 80))  # Doesn't need to be reachable\n+        s.connect((\"8.8.8.8\", 80))  # Doesn't need to be reachable\n         return s.getsockname()[0]\n     except OSError:\n         # try ipv6\n", "issue": "why vllm==0.3.3 need to access google\n![\u5fae\u4fe1\u56fe\u7247_20240304112403](https://github.com/vllm-project/vllm/assets/38678334/f21e1ec0-bd6f-4b26-aeee-d6e4e5822fc2)\r\n\n", "before_files": [{"content": "import enum\nimport os\nimport socket\nimport subprocess\nimport uuid\nfrom platform import uname\nfrom typing import List, Tuple, Union\nfrom packaging.version import parse, Version\n\nimport psutil\nimport torch\nimport asyncio\nfrom functools import partial\nfrom typing import (\n    Awaitable,\n    Callable,\n    TypeVar,\n)\nfrom collections import OrderedDict\nfrom typing import Any, Hashable, Optional\n\nfrom vllm.logger import init_logger\n\nT = TypeVar(\"T\")\nlogger = init_logger(__name__)\n\nSTR_DTYPE_TO_TORCH_DTYPE = {\n    \"half\": torch.half,\n    \"bfloat16\": torch.bfloat16,\n    \"float\": torch.float,\n    \"fp8_e5m2\": torch.uint8,\n}\n\n\nclass Device(enum.Enum):\n    GPU = enum.auto()\n    CPU = enum.auto()\n\n\nclass Counter:\n\n    def __init__(self, start: int = 0) -> None:\n        self.counter = start\n\n    def __next__(self) -> int:\n        i = self.counter\n        self.counter += 1\n        return i\n\n    def reset(self) -> None:\n        self.counter = 0\n\n\nclass LRUCache:\n\n    def __init__(self, capacity: int):\n        self.cache = OrderedDict()\n        self.capacity = capacity\n\n    def __contains__(self, key: Hashable) -> bool:\n        return key in self.cache\n\n    def __len__(self) -> int:\n        return len(self.cache)\n\n    def __getitem__(self, key: Hashable) -> Any:\n        return self.get(key)\n\n    def __setitem__(self, key: Hashable, value: Any) -> None:\n        self.put(key, value)\n\n    def __delitem__(self, key: Hashable) -> None:\n        self.pop(key)\n\n    def touch(self, key: Hashable) -> None:\n        self.cache.move_to_end(key)\n\n    def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:\n        if key in self.cache:\n            value = self.cache[key]\n            self.cache.move_to_end(key)\n        else:\n            value = default_value\n        return value\n\n    def put(self, key: Hashable, value: Any) -> None:\n        self.cache[key] = value\n        self.cache.move_to_end(key)\n        self._remove_old_if_needed()\n\n    def _on_remove(self, key: Hashable, value: Any):\n        pass\n\n    def remove_oldest(self):\n        if not self.cache:\n            return\n        key, value = self.cache.popitem(last=False)\n        self._on_remove(key, value)\n\n    def _remove_old_if_needed(self) -> None:\n        while len(self.cache) > self.capacity:\n            self.remove_oldest()\n\n    def pop(self, key: int, default_value: Optional[Any] = None) -> Any:\n        run_on_remove = key in self.cache\n        value = self.cache.pop(key, default_value)\n        if run_on_remove:\n            self._on_remove(key, value)\n        return value\n\n    def clear(self):\n        while len(self.cache) > 0:\n            self.remove_oldest()\n        self.cache.clear()\n\n\ndef is_hip() -> bool:\n    return torch.version.hip is not None\n\n\ndef is_neuron() -> bool:\n    try:\n        import transformers_neuronx\n    except ImportError:\n        transformers_neuronx = None\n    return transformers_neuronx is not None\n\n\ndef get_max_shared_memory_bytes(gpu: int = 0) -> int:\n    \"\"\"Returns the maximum shared memory per thread block in bytes.\"\"\"\n    # NOTE: This import statement should be executed lazily since\n    # the Neuron-X backend does not have the `cuda_utils` module.\n    from vllm._C import cuda_utils\n\n    max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(\n        gpu)\n    # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail\n    assert max_shared_mem > 0, \"max_shared_mem can not be zero\"\n    return int(max_shared_mem)\n\n\ndef get_cpu_memory() -> int:\n    \"\"\"Returns the total CPU memory of the node in bytes.\"\"\"\n    return psutil.virtual_memory().total\n\n\ndef random_uuid() -> str:\n    return str(uuid.uuid4().hex)\n\n\ndef in_wsl() -> bool:\n    # Reference: https://github.com/microsoft/WSL/issues/4071\n    return \"microsoft\" in \" \".join(uname()).lower()\n\n\ndef make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:\n    \"\"\"Take a blocking function, and run it on in an executor thread.\n\n    This function prevents the blocking function from blocking the\n    asyncio event loop.\n    The code in this function needs to be thread safe.\n    \"\"\"\n\n    def _async_wrapper(*args, **kwargs) -> asyncio.Future:\n        loop = asyncio.get_event_loop()\n        p_func = partial(func, *args, **kwargs)\n        return loop.run_in_executor(executor=None, func=p_func)\n\n    return _async_wrapper\n\n\ndef get_ip() -> str:\n    # try ipv4\n    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n    try:\n        s.connect((\"dns.google\", 80))  # Doesn't need to be reachable\n        return s.getsockname()[0]\n    except OSError:\n        # try ipv6\n        s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)\n        s.connect((\"dns.google\", 80))\n        return s.getsockname()[0]\n\n\ndef get_distributed_init_method(ip: str, port: int) -> str:\n    return f\"tcp://{ip}:{port}\"\n\n\ndef get_open_port() -> int:\n    # try ipv4\n    try:\n        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n            s.bind((\"\", 0))\n            return s.getsockname()[1]\n    except OSError:\n        # try ipv6\n        with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s:\n            s.bind((\"\", 0))\n            return s.getsockname()[1]\n\n\ndef set_cuda_visible_devices(device_ids: List[int]) -> None:\n    os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, device_ids))\n\n\ndef get_nvcc_cuda_version() -> Optional[Version]:\n    cuda_home = os.environ.get('CUDA_HOME')\n    if not cuda_home:\n        cuda_home = '/usr/local/cuda'\n        if os.path.isfile(cuda_home + '/bin/nvcc'):\n            logger.info(\n                f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'\n            )\n        else:\n            logger.warning(\n                f'Not found nvcc in {cuda_home}. Skip cuda version check!')\n            return None\n    nvcc_output = subprocess.check_output([cuda_home + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef _generate_random_fp8_e5m2(\n    tensor: torch.tensor,\n    low: float,\n    high: float,\n) -> None:\n    # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,\n    # it may occur Inf or NaN if we directly use torch.randint\n    # to generate random data for fp8 data.\n    # For example, s.11111.00 in fp8e5m2 format represents Inf.\n    #     | E4M3        | E5M2\n    #-----|-------------|-------------------\n    # Inf | N/A         | s.11111.00\n    # NaN | s.1111.111  | s.11111.{01,10,11}\n    from vllm._C import cache_ops\n    tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)\n    tensor_tmp.uniform_(low, high)\n    cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)\n    del tensor_tmp\n\n\ndef create_kv_caches_with_random(\n    num_blocks: int,\n    block_size: int,\n    num_layers: int,\n    num_heads: int,\n    head_size: int,\n    cache_dtype: Optional[Union[str, torch.dtype]],\n    model_dtype: Optional[Union[str, torch.dtype]] = None,\n    seed: Optional[int] = 0,\n    device: Optional[str] = \"cuda\",\n) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:\n    torch.random.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n\n    if isinstance(cache_dtype, str):\n        if cache_dtype == \"auto\":\n            if isinstance(model_dtype, str):\n                torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]\n            elif isinstance(model_dtype, torch.dtype):\n                torch_dtype = model_dtype\n            else:\n                raise ValueError(f\"Invalid model dtype: {model_dtype}\")\n        elif cache_dtype in [\"half\", \"bfloat16\", \"float\"]:\n            torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]\n        elif cache_dtype == \"fp8_e5m2\":\n            torch_dtype = torch.uint8\n        else:\n            raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n    elif isinstance(cache_dtype, torch.dtype):\n        torch_dtype = cache_dtype\n    else:\n        raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n\n    scale = head_size**-0.5\n    x = 16 // torch.tensor([], dtype=torch_dtype).element_size()\n    key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)\n    key_caches = []\n    for _ in range(num_layers):\n        key_cache = torch.empty(size=key_cache_shape,\n                                dtype=torch_dtype,\n                                device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(key_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            key_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support key cache of type {cache_dtype}\")\n        key_caches.append(key_cache)\n\n    value_cache_shape = (num_blocks, num_heads, head_size, block_size)\n    value_caches = []\n    for _ in range(num_layers):\n        value_cache = torch.empty(size=value_cache_shape,\n                                  dtype=torch_dtype,\n                                  device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(value_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            value_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support value cache of type {cache_dtype}\")\n        value_caches.append(value_cache)\n    return key_caches, value_caches\n", "path": "vllm/utils.py"}], "after_files": [{"content": "import enum\nimport os\nimport socket\nimport subprocess\nimport uuid\nfrom platform import uname\nfrom typing import List, Tuple, Union\nfrom packaging.version import parse, Version\n\nimport psutil\nimport torch\nimport asyncio\nfrom functools import partial\nfrom typing import (\n    Awaitable,\n    Callable,\n    TypeVar,\n)\nfrom collections import OrderedDict\nfrom typing import Any, Hashable, Optional\n\nfrom vllm.logger import init_logger\n\nT = TypeVar(\"T\")\nlogger = init_logger(__name__)\n\nSTR_DTYPE_TO_TORCH_DTYPE = {\n    \"half\": torch.half,\n    \"bfloat16\": torch.bfloat16,\n    \"float\": torch.float,\n    \"fp8_e5m2\": torch.uint8,\n}\n\n\nclass Device(enum.Enum):\n    GPU = enum.auto()\n    CPU = enum.auto()\n\n\nclass Counter:\n\n    def __init__(self, start: int = 0) -> None:\n        self.counter = start\n\n    def __next__(self) -> int:\n        i = self.counter\n        self.counter += 1\n        return i\n\n    def reset(self) -> None:\n        self.counter = 0\n\n\nclass LRUCache:\n\n    def __init__(self, capacity: int):\n        self.cache = OrderedDict()\n        self.capacity = capacity\n\n    def __contains__(self, key: Hashable) -> bool:\n        return key in self.cache\n\n    def __len__(self) -> int:\n        return len(self.cache)\n\n    def __getitem__(self, key: Hashable) -> Any:\n        return self.get(key)\n\n    def __setitem__(self, key: Hashable, value: Any) -> None:\n        self.put(key, value)\n\n    def __delitem__(self, key: Hashable) -> None:\n        self.pop(key)\n\n    def touch(self, key: Hashable) -> None:\n        self.cache.move_to_end(key)\n\n    def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:\n        if key in self.cache:\n            value = self.cache[key]\n            self.cache.move_to_end(key)\n        else:\n            value = default_value\n        return value\n\n    def put(self, key: Hashable, value: Any) -> None:\n        self.cache[key] = value\n        self.cache.move_to_end(key)\n        self._remove_old_if_needed()\n\n    def _on_remove(self, key: Hashable, value: Any):\n        pass\n\n    def remove_oldest(self):\n        if not self.cache:\n            return\n        key, value = self.cache.popitem(last=False)\n        self._on_remove(key, value)\n\n    def _remove_old_if_needed(self) -> None:\n        while len(self.cache) > self.capacity:\n            self.remove_oldest()\n\n    def pop(self, key: int, default_value: Optional[Any] = None) -> Any:\n        run_on_remove = key in self.cache\n        value = self.cache.pop(key, default_value)\n        if run_on_remove:\n            self._on_remove(key, value)\n        return value\n\n    def clear(self):\n        while len(self.cache) > 0:\n            self.remove_oldest()\n        self.cache.clear()\n\n\ndef is_hip() -> bool:\n    return torch.version.hip is not None\n\n\ndef is_neuron() -> bool:\n    try:\n        import transformers_neuronx\n    except ImportError:\n        transformers_neuronx = None\n    return transformers_neuronx is not None\n\n\ndef get_max_shared_memory_bytes(gpu: int = 0) -> int:\n    \"\"\"Returns the maximum shared memory per thread block in bytes.\"\"\"\n    # NOTE: This import statement should be executed lazily since\n    # the Neuron-X backend does not have the `cuda_utils` module.\n    from vllm._C import cuda_utils\n\n    max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(\n        gpu)\n    # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail\n    assert max_shared_mem > 0, \"max_shared_mem can not be zero\"\n    return int(max_shared_mem)\n\n\ndef get_cpu_memory() -> int:\n    \"\"\"Returns the total CPU memory of the node in bytes.\"\"\"\n    return psutil.virtual_memory().total\n\n\ndef random_uuid() -> str:\n    return str(uuid.uuid4().hex)\n\n\ndef in_wsl() -> bool:\n    # Reference: https://github.com/microsoft/WSL/issues/4071\n    return \"microsoft\" in \" \".join(uname()).lower()\n\n\ndef make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:\n    \"\"\"Take a blocking function, and run it on in an executor thread.\n\n    This function prevents the blocking function from blocking the\n    asyncio event loop.\n    The code in this function needs to be thread safe.\n    \"\"\"\n\n    def _async_wrapper(*args, **kwargs) -> asyncio.Future:\n        loop = asyncio.get_event_loop()\n        p_func = partial(func, *args, **kwargs)\n        return loop.run_in_executor(executor=None, func=p_func)\n\n    return _async_wrapper\n\n\ndef get_ip() -> str:\n    # try ipv4\n    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n    try:\n        s.connect((\"8.8.8.8\", 80))  # Doesn't need to be reachable\n        return s.getsockname()[0]\n    except OSError:\n        # try ipv6\n        s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)\n        s.connect((\"dns.google\", 80))\n        return s.getsockname()[0]\n\n\ndef get_distributed_init_method(ip: str, port: int) -> str:\n    return f\"tcp://{ip}:{port}\"\n\n\ndef get_open_port() -> int:\n    # try ipv4\n    try:\n        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n            s.bind((\"\", 0))\n            return s.getsockname()[1]\n    except OSError:\n        # try ipv6\n        with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s:\n            s.bind((\"\", 0))\n            return s.getsockname()[1]\n\n\ndef set_cuda_visible_devices(device_ids: List[int]) -> None:\n    os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, device_ids))\n\n\ndef get_nvcc_cuda_version() -> Optional[Version]:\n    cuda_home = os.environ.get('CUDA_HOME')\n    if not cuda_home:\n        cuda_home = '/usr/local/cuda'\n        if os.path.isfile(cuda_home + '/bin/nvcc'):\n            logger.info(\n                f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'\n            )\n        else:\n            logger.warning(\n                f'Not found nvcc in {cuda_home}. Skip cuda version check!')\n            return None\n    nvcc_output = subprocess.check_output([cuda_home + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef _generate_random_fp8_e5m2(\n    tensor: torch.tensor,\n    low: float,\n    high: float,\n) -> None:\n    # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,\n    # it may occur Inf or NaN if we directly use torch.randint\n    # to generate random data for fp8 data.\n    # For example, s.11111.00 in fp8e5m2 format represents Inf.\n    #     | E4M3        | E5M2\n    #-----|-------------|-------------------\n    # Inf | N/A         | s.11111.00\n    # NaN | s.1111.111  | s.11111.{01,10,11}\n    from vllm._C import cache_ops\n    tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)\n    tensor_tmp.uniform_(low, high)\n    cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)\n    del tensor_tmp\n\n\ndef create_kv_caches_with_random(\n    num_blocks: int,\n    block_size: int,\n    num_layers: int,\n    num_heads: int,\n    head_size: int,\n    cache_dtype: Optional[Union[str, torch.dtype]],\n    model_dtype: Optional[Union[str, torch.dtype]] = None,\n    seed: Optional[int] = 0,\n    device: Optional[str] = \"cuda\",\n) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:\n    torch.random.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n\n    if isinstance(cache_dtype, str):\n        if cache_dtype == \"auto\":\n            if isinstance(model_dtype, str):\n                torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]\n            elif isinstance(model_dtype, torch.dtype):\n                torch_dtype = model_dtype\n            else:\n                raise ValueError(f\"Invalid model dtype: {model_dtype}\")\n        elif cache_dtype in [\"half\", \"bfloat16\", \"float\"]:\n            torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]\n        elif cache_dtype == \"fp8_e5m2\":\n            torch_dtype = torch.uint8\n        else:\n            raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n    elif isinstance(cache_dtype, torch.dtype):\n        torch_dtype = cache_dtype\n    else:\n        raise ValueError(f\"Invalid kv cache dtype: {cache_dtype}\")\n\n    scale = head_size**-0.5\n    x = 16 // torch.tensor([], dtype=torch_dtype).element_size()\n    key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)\n    key_caches = []\n    for _ in range(num_layers):\n        key_cache = torch.empty(size=key_cache_shape,\n                                dtype=torch_dtype,\n                                device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(key_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            key_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support key cache of type {cache_dtype}\")\n        key_caches.append(key_cache)\n\n    value_cache_shape = (num_blocks, num_heads, head_size, block_size)\n    value_caches = []\n    for _ in range(num_layers):\n        value_cache = torch.empty(size=value_cache_shape,\n                                  dtype=torch_dtype,\n                                  device=device)\n        if cache_dtype == 'fp8_e5m2':\n            _generate_random_fp8_e5m2(value_cache, -scale, scale)\n        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:\n            value_cache.uniform_(-scale, scale)\n        else:\n            raise ValueError(\n                f\"Does not support value cache of type {cache_dtype}\")\n        value_caches.append(value_cache)\n    return key_caches, value_caches\n", "path": "vllm/utils.py"}]}