We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Support both CUDA 11.8 and CUDA 12.1
Hi vLLM maintainers. I suggest maintaining compatibility with torch 2.0.1 and CUDA 11.8.0 for a few more versions. The way this would work is that you create two versions of the wheel:
- PyPi: torch 2.1.0 and CUDA 12.1.1 wheel
- GitHub release: additional torch 2.0.1 and CUDA 11.8.0 wheel

The idea is that people can still install vLLM from the GitHub release if they do not have the latest CUDA version yet. This can be achieved by the following:

```python
VLLM_VERSION = "0.2.2"
PYPI_BUILD = os.getenv("PYPI_BUILD", "0") == "1"

if not PYPI_BUILD:
    try:
        CUDA_VERSION = "".join(os.environ.get("CUDA_VERSION", torch.version.cuda).split("."))[:3]
        VLLM_VERSION += f"+cu{CUDA_VERSION}"
    except Exception as ex:
        raise RuntimeError("Your system must have an Nvidia GPU for installing vLLM")
```

In the GitHub workflow, add a conditional on which and check if the current CUDA version being used to build is the same as the one you want to release on PyPi.

```
if ( $env:CUDA_VERSION -eq $env:PYPI_CUDA_VERSION ){
    $env:PYPI_BUILD = 1
}
```

For reference, you can look at `setup.py` and `build.yaml` in AutoAWQ.
https://github.com/casper-hansen/AutoAWQ/blob/main/.github/workflows/build.yaml

_Originally posted by @casper-hansen in https://github.com/vllm-project/vllm/issues/1527#issuecomment-1799749890_
            

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 import io
2 import os
3 import re
4 import subprocess
5 from typing import List, Set
6 import warnings
7 
8 from packaging.version import parse, Version
9 import setuptools
10 import torch
11 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
12 
13 ROOT_DIR = os.path.dirname(__file__)
14 
15 # Supported NVIDIA GPU architectures.
16 SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
17 
18 # Compiler flags.
19 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
20 # TODO(woosuk): Should we use -O3?
21 NVCC_FLAGS = ["-O2", "-std=c++17"]
22 
23 ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
24 CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
25 NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
26 
27 if CUDA_HOME is None:
28     raise RuntimeError(
29         "Cannot find CUDA_HOME. CUDA must be available to build the package.")
30 
31 
32 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
33     """Get the CUDA version from nvcc.
34 
35     Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
36     """
37     nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
38                                           universal_newlines=True)
39     output = nvcc_output.split()
40     release_idx = output.index("release") + 1
41     nvcc_cuda_version = parse(output[release_idx].split(",")[0])
42     return nvcc_cuda_version
43 
44 
45 def get_torch_arch_list() -> Set[str]:
46     # TORCH_CUDA_ARCH_LIST can have one or more architectures,
47     # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
48     # compiler to additionally include PTX code that can be runtime-compiled
49     # and executed on the 8.6 or newer architectures. While the PTX code will
50     # not give the best performance on the newer architectures, it provides
51     # forward compatibility.
52     env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
53     if env_arch_list is None:
54         return set()
55 
56     # List are separated by ; or space.
57     torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
58     if not torch_arch_list:
59         return set()
60 
61     # Filter out the invalid architectures and print a warning.
62     valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
63     arch_list = torch_arch_list.intersection(valid_archs)
64     # If none of the specified architectures are valid, raise an error.
65     if not arch_list:
66         raise RuntimeError(
67             "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
68             f"variable ({env_arch_list}) is supported. "
69             f"Supported CUDA architectures are: {valid_archs}.")
70     invalid_arch_list = torch_arch_list - valid_archs
71     if invalid_arch_list:
72         warnings.warn(
73             f"Unsupported CUDA architectures ({invalid_arch_list}) are "
74             "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
75             f"({env_arch_list}). Supported CUDA architectures are: "
76             f"{valid_archs}.")
77     return arch_list
78 
79 
80 # First, check the TORCH_CUDA_ARCH_LIST environment variable.
81 compute_capabilities = get_torch_arch_list()
82 if not compute_capabilities:
83     # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
84     # GPUs on the current machine.
85     device_count = torch.cuda.device_count()
86     for i in range(device_count):
87         major, minor = torch.cuda.get_device_capability(i)
88         if major < 7:
89             raise RuntimeError(
90                 "GPUs with compute capability below 7.0 are not supported.")
91         compute_capabilities.add(f"{major}.{minor}")
92 
93 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
94 if not compute_capabilities:
95     # If no GPU is specified nor available, add all supported architectures
96     # based on the NVCC CUDA version.
97     compute_capabilities = SUPPORTED_ARCHS.copy()
98     if nvcc_cuda_version < Version("11.1"):
99         compute_capabilities.remove("8.6")
100     if nvcc_cuda_version < Version("11.8"):
101         compute_capabilities.remove("8.9")
102         compute_capabilities.remove("9.0")
103 
104 # Validate the NVCC CUDA version.
105 if nvcc_cuda_version < Version("11.0"):
106     raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
107 if nvcc_cuda_version < Version("11.1"):
108     if any(cc.startswith("8.6") for cc in compute_capabilities):
109         raise RuntimeError(
110             "CUDA 11.1 or higher is required for compute capability 8.6.")
111 if nvcc_cuda_version < Version("11.8"):
112     if any(cc.startswith("8.9") for cc in compute_capabilities):
113         # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
114         # However, GPUs with compute capability 8.9 can also run the code generated by
115         # the previous versions of CUDA 11 and targeting compute capability 8.0.
116         # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
117         # instead of 8.9.
118         warnings.warn(
119             "CUDA 11.8 or higher is required for compute capability 8.9. "
120             "Targeting compute capability 8.0 instead.")
121         compute_capabilities = set(cc for cc in compute_capabilities
122                                    if not cc.startswith("8.9"))
123         compute_capabilities.add("8.0+PTX")
124     if any(cc.startswith("9.0") for cc in compute_capabilities):
125         raise RuntimeError(
126             "CUDA 11.8 or higher is required for compute capability 9.0.")
127 
128 # Add target compute capabilities to NVCC flags.
129 for capability in compute_capabilities:
130     num = capability[0] + capability[2]
131     NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
132     if capability.endswith("+PTX"):
133         NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]
134 
135 # Use NVCC threads to parallelize the build.
136 if nvcc_cuda_version >= Version("11.2"):
137     num_threads = min(os.cpu_count(), 8)
138     NVCC_FLAGS += ["--threads", str(num_threads)]
139 
140 ext_modules = []
141 
142 # Cache operations.
143 cache_extension = CUDAExtension(
144     name="vllm.cache_ops",
145     sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
146     extra_compile_args={
147         "cxx": CXX_FLAGS,
148         "nvcc": NVCC_FLAGS,
149     },
150 )
151 ext_modules.append(cache_extension)
152 
153 # Attention kernels.
154 attention_extension = CUDAExtension(
155     name="vllm.attention_ops",
156     sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
157     extra_compile_args={
158         "cxx": CXX_FLAGS,
159         "nvcc": NVCC_FLAGS,
160     },
161 )
162 ext_modules.append(attention_extension)
163 
164 # Positional encoding kernels.
165 positional_encoding_extension = CUDAExtension(
166     name="vllm.pos_encoding_ops",
167     sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
168     extra_compile_args={
169         "cxx": CXX_FLAGS,
170         "nvcc": NVCC_FLAGS,
171     },
172 )
173 ext_modules.append(positional_encoding_extension)
174 
175 # Layer normalization kernels.
176 layernorm_extension = CUDAExtension(
177     name="vllm.layernorm_ops",
178     sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
179     extra_compile_args={
180         "cxx": CXX_FLAGS,
181         "nvcc": NVCC_FLAGS,
182     },
183 )
184 ext_modules.append(layernorm_extension)
185 
186 # Activation kernels.
187 activation_extension = CUDAExtension(
188     name="vllm.activation_ops",
189     sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
190     extra_compile_args={
191         "cxx": CXX_FLAGS,
192         "nvcc": NVCC_FLAGS,
193     },
194 )
195 ext_modules.append(activation_extension)
196 
197 # Quantization kernels.
198 quantization_extension = CUDAExtension(
199     name="vllm.quantization_ops",
200     sources=[
201         "csrc/quantization.cpp",
202         "csrc/quantization/awq/gemm_kernels.cu",
203         "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
204     ],
205     extra_compile_args={
206         "cxx": CXX_FLAGS,
207         "nvcc": NVCC_FLAGS,
208     },
209 )
210 ext_modules.append(quantization_extension)
211 
212 # Misc. CUDA utils.
213 cuda_utils_extension = CUDAExtension(
214     name="vllm.cuda_utils",
215     sources=["csrc/cuda_utils.cpp", "csrc/cuda_utils_kernels.cu"],
216     extra_compile_args={
217         "cxx": CXX_FLAGS,
218         "nvcc": NVCC_FLAGS,
219     },
220 )
221 ext_modules.append(cuda_utils_extension)
222 
223 
224 def get_path(*filepath) -> str:
225     return os.path.join(ROOT_DIR, *filepath)
226 
227 
228 def find_version(filepath: str):
229     """Extract version information from the given filepath.
230 
231     Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
232     """
233     with open(filepath) as fp:
234         version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
235                                   fp.read(), re.M)
236         if version_match:
237             return version_match.group(1)
238         raise RuntimeError("Unable to find version string.")
239 
240 
241 def read_readme() -> str:
242     """Read the README file if present."""
243     p = get_path("README.md")
244     if os.path.isfile(p):
245         return io.open(get_path("README.md"), "r", encoding="utf-8").read()
246     else:
247         return ""
248 
249 
250 def get_requirements() -> List[str]:
251     """Get Python package dependencies from requirements.txt."""
252     with open(get_path("requirements.txt")) as f:
253         requirements = f.read().strip().split("\n")
254     return requirements
255 
256 
257 setuptools.setup(
258     name="vllm",
259     version=find_version(get_path("vllm", "__init__.py")),
260     author="vLLM Team",
261     license="Apache 2.0",
262     description=("A high-throughput and memory-efficient inference and "
263                  "serving engine for LLMs"),
264     long_description=read_readme(),
265     long_description_content_type="text/markdown",
266     url="https://github.com/vllm-project/vllm",
267     project_urls={
268         "Homepage": "https://github.com/vllm-project/vllm",
269         "Documentation": "https://vllm.readthedocs.io/en/latest/",
270     },
271     classifiers=[
272         "Programming Language :: Python :: 3.8",
273         "Programming Language :: Python :: 3.9",
274         "Programming Language :: Python :: 3.10",
275         "Programming Language :: Python :: 3.11",
276         "License :: OSI Approved :: Apache Software License",
277         "Topic :: Scientific/Engineering :: Artificial Intelligence",
278     ],
279     packages=setuptools.find_packages(exclude=("benchmarks", "csrc", "docs",
280                                                "examples", "tests")),
281     python_requires=">=3.8",
282     install_requires=get_requirements(),
283     ext_modules=ext_modules,
284     cmdclass={"build_ext": BuildExtension},
285     package_data={"vllm": ["py.typed"]},
286 )
287
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -12,6 +12,8 @@
 
 ROOT_DIR = os.path.dirname(__file__)
 
+MAIN_CUDA_VERSION = "12.1"
+
 # Supported NVIDIA GPU architectures.
 SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
 
@@ -225,7 +227,7 @@
     return os.path.join(ROOT_DIR, *filepath)
 
 
-def find_version(filepath: str):
+def find_version(filepath: str) -> str:
     """Extract version information from the given filepath.
 
     Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
@@ -238,6 +240,15 @@
         raise RuntimeError("Unable to find version string.")
 
 
+def get_vllm_version() -> str:
+    version = find_version(get_path("vllm", "__init__.py"))
+    cuda_version = str(nvcc_cuda_version)
+    if cuda_version != MAIN_CUDA_VERSION:
+        cuda_version_str = cuda_version.replace(".", "")[:3]
+        version += f"+cu{cuda_version_str}"
+    return version
+
+
 def read_readme() -> str:
     """Read the README file if present."""
     p = get_path("README.md")
@@ -256,7 +267,7 @@
 
 setuptools.setup(
     name="vllm",
-    version=find_version(get_path("vllm", "__init__.py")),
+    version=get_vllm_version(),
     author="vLLM Team",
     license="Apache 2.0",
     description=("A high-throughput and memory-efficient inference and "

{"golden_diff": "diff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -12,6 +12,8 @@\n \n ROOT_DIR = os.path.dirname(__file__)\n \n+MAIN_CUDA_VERSION = \"12.1\"\n+\n # Supported NVIDIA GPU architectures.\n SUPPORTED_ARCHS = {\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"}\n \n@@ -225,7 +227,7 @@\n     return os.path.join(ROOT_DIR, *filepath)\n \n \n-def find_version(filepath: str):\n+def find_version(filepath: str) -> str:\n     \"\"\"Extract version information from the given filepath.\n \n     Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n@@ -238,6 +240,15 @@\n         raise RuntimeError(\"Unable to find version string.\")\n \n \n+def get_vllm_version() -> str:\n+    version = find_version(get_path(\"vllm\", \"__init__.py\"))\n+    cuda_version = str(nvcc_cuda_version)\n+    if cuda_version != MAIN_CUDA_VERSION:\n+        cuda_version_str = cuda_version.replace(\".\", \"\")[:3]\n+        version += f\"+cu{cuda_version_str}\"\n+    return version\n+\n+\n def read_readme() -> str:\n     \"\"\"Read the README file if present.\"\"\"\n     p = get_path(\"README.md\")\n@@ -256,7 +267,7 @@\n \n setuptools.setup(\n     name=\"vllm\",\n-    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n+    version=get_vllm_version(),\n     author=\"vLLM Team\",\n     license=\"Apache 2.0\",\n     description=(\"A high-throughput and memory-efficient inference and \"\n", "issue": "Support both CUDA 11.8 and CUDA 12.1\nHi vLLM maintainers. I suggest maintaining compatibility with torch 2.0.1 and CUDA 11.8.0 for a few more versions. The way this would work is that you create two versions of the wheel:\r\n- PyPi: torch 2.1.0 and CUDA 12.1.1 wheel\r\n- GitHub release: additional torch 2.0.1 and CUDA 11.8.0 wheel\r\n\r\nThe idea is that people can still install vLLM from the GitHub release if they do not have the latest CUDA version yet. This can be achieved by the following:\r\n\r\n```python\r\nVLLM_VERSION = \"0.2.2\"\r\nPYPI_BUILD = os.getenv(\"PYPI_BUILD\", \"0\") == \"1\"\r\n\r\nif not PYPI_BUILD:\r\n    try:\r\n        CUDA_VERSION = \"\".join(os.environ.get(\"CUDA_VERSION\", torch.version.cuda).split(\".\"))[:3]\r\n        VLLM_VERSION += f\"+cu{CUDA_VERSION}\"\r\n    except Exception as ex:\r\n        raise RuntimeError(\"Your system must have an Nvidia GPU for installing vLLM\")\r\n```\r\n\r\nIn the GitHub workflow, add a conditional on which and check if the current CUDA version being used to build is the same as the one you want to release on PyPi.\r\n\r\n```\r\nif ( $env:CUDA_VERSION -eq $env:PYPI_CUDA_VERSION ){\r\n    $env:PYPI_BUILD = 1\r\n}\r\n```\r\n\r\nFor reference, you can look at `setup.py` and `build.yaml` in AutoAWQ.\r\nhttps://github.com/casper-hansen/AutoAWQ/blob/main/.github/workflows/build.yaml\r\n\r\n_Originally posted by @casper-hansen in https://github.com/vllm-project/vllm/issues/1527#issuecomment-1799749890_\r\n            \n", "before_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\nimport warnings\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\n# Supported NVIDIA GPU architectures.\nSUPPORTED_ARCHS = {\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"}\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        \"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef get_torch_arch_list() -> Set[str]:\n    # TORCH_CUDA_ARCH_LIST can have one or more architectures,\n    # e.g. \"8.0\" or \"7.5,8.0,8.6+PTX\". Here, the \"8.6+PTX\" option asks the\n    # compiler to additionally include PTX code that can be runtime-compiled\n    # and executed on the 8.6 or newer architectures. While the PTX code will\n    # not give the best performance on the newer architectures, it provides\n    # forward compatibility.\n    env_arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n    if env_arch_list is None:\n        return set()\n\n    # List are separated by ; or space.\n    torch_arch_list = set(env_arch_list.replace(\" \", \";\").split(\";\"))\n    if not torch_arch_list:\n        return set()\n\n    # Filter out the invalid architectures and print a warning.\n    valid_archs = SUPPORTED_ARCHS.union({s + \"+PTX\" for s in SUPPORTED_ARCHS})\n    arch_list = torch_arch_list.intersection(valid_archs)\n    # If none of the specified architectures are valid, raise an error.\n    if not arch_list:\n        raise RuntimeError(\n            \"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env \"\n            f\"variable ({env_arch_list}) is supported. \"\n            f\"Supported CUDA architectures are: {valid_archs}.\")\n    invalid_arch_list = torch_arch_list - valid_archs\n    if invalid_arch_list:\n        warnings.warn(\n            f\"Unsupported CUDA architectures ({invalid_arch_list}) are \"\n            \"excluded from the `TORCH_CUDA_ARCH_LIST` env variable \"\n            f\"({env_arch_list}). Supported CUDA architectures are: \"\n            f\"{valid_archs}.\")\n    return arch_list\n\n\n# First, check the TORCH_CUDA_ARCH_LIST environment variable.\ncompute_capabilities = get_torch_arch_list()\nif not compute_capabilities:\n    # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available\n    # GPUs on the current machine.\n    device_count = torch.cuda.device_count()\n    for i in range(device_count):\n        major, minor = torch.cuda.get_device_capability(i)\n        if major < 7:\n            raise RuntimeError(\n                \"GPUs with compute capability below 7.0 are not supported.\")\n        compute_capabilities.add(f\"{major}.{minor}\")\n\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif not compute_capabilities:\n    # If no GPU is specified nor available, add all supported architectures\n    # based on the NVCC CUDA version.\n    compute_capabilities = SUPPORTED_ARCHS.copy()\n    if nvcc_cuda_version < Version(\"11.1\"):\n        compute_capabilities.remove(\"8.6\")\n    if nvcc_cuda_version < Version(\"11.8\"):\n        compute_capabilities.remove(\"8.9\")\n        compute_capabilities.remove(\"9.0\")\n\n# Validate the NVCC CUDA version.\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif nvcc_cuda_version < Version(\"11.1\"):\n    if any(cc.startswith(\"8.6\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.1 or higher is required for compute capability 8.6.\")\nif nvcc_cuda_version < Version(\"11.8\"):\n    if any(cc.startswith(\"8.9\") for cc in compute_capabilities):\n        # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n        # However, GPUs with compute capability 8.9 can also run the code generated by\n        # the previous versions of CUDA 11 and targeting compute capability 8.0.\n        # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n        # instead of 8.9.\n        warnings.warn(\n            \"CUDA 11.8 or higher is required for compute capability 8.9. \"\n            \"Targeting compute capability 8.0 instead.\")\n        compute_capabilities = set(cc for cc in compute_capabilities\n                                   if not cc.startswith(\"8.9\"))\n        compute_capabilities.add(\"8.0+PTX\")\n    if any(cc.startswith(\"9.0\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.8 or higher is required for compute capability 9.0.\")\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    num = capability[0] + capability[2]\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=sm_{num}\"]\n    if capability.endswith(\"+PTX\"):\n        NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=compute_{num}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(activation_extension)\n\n# Quantization kernels.\nquantization_extension = CUDAExtension(\n    name=\"vllm.quantization_ops\",\n    sources=[\n        \"csrc/quantization.cpp\",\n        \"csrc/quantization/awq/gemm_kernels.cu\",\n        \"csrc/quantization/squeezellm/quant_cuda_kernel.cu\",\n    ],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(quantization_extension)\n\n# Misc. CUDA utils.\ncuda_utils_extension = CUDAExtension(\n    name=\"vllm.cuda_utils\",\n    sources=[\"csrc/cuda_utils.cpp\", \"csrc/cuda_utils_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cuda_utils_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str):\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file if present.\"\"\"\n    p = get_path(\"README.md\")\n    if os.path.isfile(p):\n        return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n    else:\n        return \"\"\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=(\"A high-throughput and memory-efficient inference and \"\n                 \"serving engine for LLMs\"),\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"Programming Language :: Python :: 3.11\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(exclude=(\"benchmarks\", \"csrc\", \"docs\",\n                                               \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n    package_data={\"vllm\": [\"py.typed\"]},\n)\n", "path": "setup.py"}], "after_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\nimport warnings\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\nMAIN_CUDA_VERSION = \"12.1\"\n\n# Supported NVIDIA GPU architectures.\nSUPPORTED_ARCHS = {\"7.0\", \"7.5\", \"8.0\", \"8.6\", \"8.9\", \"9.0\"}\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        \"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\ndef get_torch_arch_list() -> Set[str]:\n    # TORCH_CUDA_ARCH_LIST can have one or more architectures,\n    # e.g. \"8.0\" or \"7.5,8.0,8.6+PTX\". Here, the \"8.6+PTX\" option asks the\n    # compiler to additionally include PTX code that can be runtime-compiled\n    # and executed on the 8.6 or newer architectures. While the PTX code will\n    # not give the best performance on the newer architectures, it provides\n    # forward compatibility.\n    env_arch_list = os.environ.get(\"TORCH_CUDA_ARCH_LIST\", None)\n    if env_arch_list is None:\n        return set()\n\n    # List are separated by ; or space.\n    torch_arch_list = set(env_arch_list.replace(\" \", \";\").split(\";\"))\n    if not torch_arch_list:\n        return set()\n\n    # Filter out the invalid architectures and print a warning.\n    valid_archs = SUPPORTED_ARCHS.union({s + \"+PTX\" for s in SUPPORTED_ARCHS})\n    arch_list = torch_arch_list.intersection(valid_archs)\n    # If none of the specified architectures are valid, raise an error.\n    if not arch_list:\n        raise RuntimeError(\n            \"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env \"\n            f\"variable ({env_arch_list}) is supported. \"\n            f\"Supported CUDA architectures are: {valid_archs}.\")\n    invalid_arch_list = torch_arch_list - valid_archs\n    if invalid_arch_list:\n        warnings.warn(\n            f\"Unsupported CUDA architectures ({invalid_arch_list}) are \"\n            \"excluded from the `TORCH_CUDA_ARCH_LIST` env variable \"\n            f\"({env_arch_list}). Supported CUDA architectures are: \"\n            f\"{valid_archs}.\")\n    return arch_list\n\n\n# First, check the TORCH_CUDA_ARCH_LIST environment variable.\ncompute_capabilities = get_torch_arch_list()\nif not compute_capabilities:\n    # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available\n    # GPUs on the current machine.\n    device_count = torch.cuda.device_count()\n    for i in range(device_count):\n        major, minor = torch.cuda.get_device_capability(i)\n        if major < 7:\n            raise RuntimeError(\n                \"GPUs with compute capability below 7.0 are not supported.\")\n        compute_capabilities.add(f\"{major}.{minor}\")\n\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif not compute_capabilities:\n    # If no GPU is specified nor available, add all supported architectures\n    # based on the NVCC CUDA version.\n    compute_capabilities = SUPPORTED_ARCHS.copy()\n    if nvcc_cuda_version < Version(\"11.1\"):\n        compute_capabilities.remove(\"8.6\")\n    if nvcc_cuda_version < Version(\"11.8\"):\n        compute_capabilities.remove(\"8.9\")\n        compute_capabilities.remove(\"9.0\")\n\n# Validate the NVCC CUDA version.\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif nvcc_cuda_version < Version(\"11.1\"):\n    if any(cc.startswith(\"8.6\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.1 or higher is required for compute capability 8.6.\")\nif nvcc_cuda_version < Version(\"11.8\"):\n    if any(cc.startswith(\"8.9\") for cc in compute_capabilities):\n        # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n        # However, GPUs with compute capability 8.9 can also run the code generated by\n        # the previous versions of CUDA 11 and targeting compute capability 8.0.\n        # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n        # instead of 8.9.\n        warnings.warn(\n            \"CUDA 11.8 or higher is required for compute capability 8.9. \"\n            \"Targeting compute capability 8.0 instead.\")\n        compute_capabilities = set(cc for cc in compute_capabilities\n                                   if not cc.startswith(\"8.9\"))\n        compute_capabilities.add(\"8.0+PTX\")\n    if any(cc.startswith(\"9.0\") for cc in compute_capabilities):\n        raise RuntimeError(\n            \"CUDA 11.8 or higher is required for compute capability 9.0.\")\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    num = capability[0] + capability[2]\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=sm_{num}\"]\n    if capability.endswith(\"+PTX\"):\n        NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{num},code=compute_{num}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(activation_extension)\n\n# Quantization kernels.\nquantization_extension = CUDAExtension(\n    name=\"vllm.quantization_ops\",\n    sources=[\n        \"csrc/quantization.cpp\",\n        \"csrc/quantization/awq/gemm_kernels.cu\",\n        \"csrc/quantization/squeezellm/quant_cuda_kernel.cu\",\n    ],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(quantization_extension)\n\n# Misc. CUDA utils.\ncuda_utils_extension = CUDAExtension(\n    name=\"vllm.cuda_utils\",\n    sources=[\"csrc/cuda_utils.cpp\", \"csrc/cuda_utils_kernels.cu\"],\n    extra_compile_args={\n        \"cxx\": CXX_FLAGS,\n        \"nvcc\": NVCC_FLAGS,\n    },\n)\next_modules.append(cuda_utils_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str) -> str:\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef get_vllm_version() -> str:\n    version = find_version(get_path(\"vllm\", \"__init__.py\"))\n    cuda_version = str(nvcc_cuda_version)\n    if cuda_version != MAIN_CUDA_VERSION:\n        cuda_version_str = cuda_version.replace(\".\", \"\")[:3]\n        version += f\"+cu{cuda_version_str}\"\n    return version\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file if present.\"\"\"\n    p = get_path(\"README.md\")\n    if os.path.isfile(p):\n        return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n    else:\n        return \"\"\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=get_vllm_version(),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=(\"A high-throughput and memory-efficient inference and \"\n                 \"serving engine for LLMs\"),\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"Programming Language :: Python :: 3.11\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(exclude=(\"benchmarks\", \"csrc\", \"docs\",\n                                               \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n    package_data={\"vllm\": [\"py.typed\"]},\n)\n", "path": "setup.py"}]}