We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
 Issue Building vllm Wheel: Unsupported GPU Architecture 'compute_89'
I'm trying to build the vllm package from source (pip install vllm), but I'm encountering an error related to CUDA and GPU architecture. I am using RTX 4090. Here are the crucial parts of the log:


```
Building wheels for collected packages: vllm
Building wheel for vllm (pyproject.toml) ... error
error: subprocess-exited-with-error

...

building 'vllm.cache_ops' extension
Emitting ninja build file ...

...

/usr/local/cuda/bin/nvcc  -I/tmp/pip-build-env-... (other flags and paths)
nvcc fatal   : Unsupported gpu architecture 'compute_89'
```

Here is my compute cluster:
---
6790722
Age:15 min.[Remaining](https://vast.ai/faq#Instances):1d 19h71.69.80.64:44536-44750North Carolina, US1x RTX 409081.8  TFLOPS
m:9663host:39244verified0.3/24.6 GB
3506.9 GB/s
ROME2D16
PCIE 4.0, 8x12.5 GB/s
AMD EPYC 7662 64-Core Processor
25.6/256 cpu
4/52 GB
nvme
4972 MB/s6.9/21.0 GB
516.4 Mbps514.0 MbpsMax CUDA: 12.2
 
GPU: 0% 35C , CPU: 4% Status: success, running pytorch/pytorch_2.0.1-cuda11.7-cudnn8-devel/jupyter
$0.263/hr 
Current State : running Next State : running
Docker Image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel

Image Runtype : jupyter_direc ssh_direc ssh_proxy

Environment :
{
  "JUPYTER_DIR": "/"
}

Docker Args:

Onstart: env | grep _ >> /etc/environment; echo 'starting up'


---

It seems that the nvcc compiler doesn't support the 'compute_89' GPU architecture. I'm using CUDA version 11.7 on a RTX 4090.

Has anyone encountered this issue before? Any suggestions on how to resolve it?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 import io
2 import os
3 import re
4 import subprocess
5 from typing import List, Set
6 
7 from packaging.version import parse, Version
8 import setuptools
9 import torch
10 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
11 
12 ROOT_DIR = os.path.dirname(__file__)
13 
14 # Compiler flags.
15 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
16 # TODO(woosuk): Should we use -O3?
17 NVCC_FLAGS = ["-O2", "-std=c++17"]
18 
19 ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
20 CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
21 NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22 
23 if CUDA_HOME is None:
24     raise RuntimeError(
25         f"Cannot find CUDA_HOME. CUDA must be available in order to build the package.")
26 
27 
28 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
29     """Get the CUDA version from nvcc.
30 
31     Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
32     """
33     nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
34                                           universal_newlines=True)
35     output = nvcc_output.split()
36     release_idx = output.index("release") + 1
37     nvcc_cuda_version = parse(output[release_idx].split(",")[0])
38     return nvcc_cuda_version
39 
40 
41 # Collect the compute capabilities of all available GPUs.
42 device_count = torch.cuda.device_count()
43 compute_capabilities: Set[int] = set()
44 for i in range(device_count):
45     major, minor = torch.cuda.get_device_capability(i)
46     if major < 7:
47         raise RuntimeError(
48             "GPUs with compute capability less than 7.0 are not supported.")
49     compute_capabilities.add(major * 10 + minor)
50 
51 # Validate the NVCC CUDA version.
52 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
53 if nvcc_cuda_version < Version("11.0"):
54     raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55 if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56     raise RuntimeError(
57         "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
58 if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59     raise RuntimeError(
60         "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
61 
62 # If no GPU is available, add all supported compute capabilities.
63 if not compute_capabilities:
64     compute_capabilities = {70, 75, 80}
65     if nvcc_cuda_version >= Version("11.1"):
66         compute_capabilities.add(86)
67     if nvcc_cuda_version >= Version("11.8"):
68         compute_capabilities.add(90)
69 
70 # Add target compute capabilities to NVCC flags.
71 for capability in compute_capabilities:
72     NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
73 
74 # Use NVCC threads to parallelize the build.
75 if nvcc_cuda_version >= Version("11.2"):
76     num_threads = min(os.cpu_count(), 8)
77     NVCC_FLAGS += ["--threads", str(num_threads)]
78 
79 ext_modules = []
80 
81 # Cache operations.
82 cache_extension = CUDAExtension(
83     name="vllm.cache_ops",
84     sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
85     extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
86 )
87 ext_modules.append(cache_extension)
88 
89 # Attention kernels.
90 attention_extension = CUDAExtension(
91     name="vllm.attention_ops",
92     sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
93     extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
94 )
95 ext_modules.append(attention_extension)
96 
97 # Positional encoding kernels.
98 positional_encoding_extension = CUDAExtension(
99     name="vllm.pos_encoding_ops",
100     sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
101     extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
102 )
103 ext_modules.append(positional_encoding_extension)
104 
105 # Layer normalization kernels.
106 layernorm_extension = CUDAExtension(
107     name="vllm.layernorm_ops",
108     sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
109     extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
110 )
111 ext_modules.append(layernorm_extension)
112 
113 # Activation kernels.
114 activation_extension = CUDAExtension(
115     name="vllm.activation_ops",
116     sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
117     extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
118 )
119 ext_modules.append(activation_extension)
120 
121 
122 def get_path(*filepath) -> str:
123     return os.path.join(ROOT_DIR, *filepath)
124 
125 
126 def find_version(filepath: str):
127     """Extract version information from the given filepath.
128 
129     Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
130     """
131     with open(filepath) as fp:
132         version_match = re.search(
133             r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
134         if version_match:
135             return version_match.group(1)
136         raise RuntimeError("Unable to find version string.")
137 
138 
139 def read_readme() -> str:
140     """Read the README file."""
141     return io.open(get_path("README.md"), "r", encoding="utf-8").read()
142 
143 
144 def get_requirements() -> List[str]:
145     """Get Python package dependencies from requirements.txt."""
146     with open(get_path("requirements.txt")) as f:
147         requirements = f.read().strip().split("\n")
148     return requirements
149 
150 
151 setuptools.setup(
152     name="vllm",
153     version=find_version(get_path("vllm", "__init__.py")),
154     author="vLLM Team",
155     license="Apache 2.0",
156     description="A high-throughput and memory-efficient inference and serving engine for LLMs",
157     long_description=read_readme(),
158     long_description_content_type="text/markdown",
159     url="https://github.com/vllm-project/vllm",
160     project_urls={
161         "Homepage": "https://github.com/vllm-project/vllm",
162         "Documentation": "https://vllm.readthedocs.io/en/latest/",
163     },
164     classifiers=[
165         "Programming Language :: Python :: 3.8",
166         "Programming Language :: Python :: 3.9",
167         "Programming Language :: Python :: 3.10",
168         "License :: OSI Approved :: Apache Software License",
169         "Topic :: Scientific/Engineering :: Artificial Intelligence",
170     ],
171     packages=setuptools.find_packages(
172         exclude=("assets", "benchmarks", "csrc", "docs", "examples", "tests")),
173     python_requires=">=3.8",
174     install_requires=get_requirements(),
175     ext_modules=ext_modules,
176     cmdclass={"build_ext": BuildExtension},
177 )
178
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
 
 if CUDA_HOME is None:
     raise RuntimeError(
-        f"Cannot find CUDA_HOME. CUDA must be available in order to build the package.")
+        f"Cannot find CUDA_HOME. CUDA must be available to build the package.")
 
 
 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -55,6 +55,14 @@
 if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
     raise RuntimeError(
         "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
+if 89 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
+    # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
+    # However, GPUs with compute capability 8.9 can also run the code generated by
+    # the previous versions of CUDA 11 and targeting compute capability 8.0.
+    # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
+    # instead of 8.9.
+    compute_capabilities.remove(89)
+    compute_capabilities.add(80)
 if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
     raise RuntimeError(
         "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
@@ -65,6 +73,7 @@
     if nvcc_cuda_version >= Version("11.1"):
         compute_capabilities.add(86)
     if nvcc_cuda_version >= Version("11.8"):
+        compute_capabilities.add(89)
         compute_capabilities.add(90)
 
 # Add target compute capabilities to NVCC flags.

{"golden_diff": "diff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -22,7 +22,7 @@\n \n if CUDA_HOME is None:\n     raise RuntimeError(\n-        f\"Cannot find CUDA_HOME. CUDA must be available in order to build the package.\")\n+        f\"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n \n \n def get_nvcc_cuda_version(cuda_dir: str) -> Version:\n@@ -55,6 +55,14 @@\n if 86 in compute_capabilities and nvcc_cuda_version < Version(\"11.1\"):\n     raise RuntimeError(\n         \"CUDA 11.1 or higher is required for GPUs with compute capability 8.6.\")\n+if 89 in compute_capabilities and nvcc_cuda_version < Version(\"11.8\"):\n+    # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n+    # However, GPUs with compute capability 8.9 can also run the code generated by\n+    # the previous versions of CUDA 11 and targeting compute capability 8.0.\n+    # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n+    # instead of 8.9.\n+    compute_capabilities.remove(89)\n+    compute_capabilities.add(80)\n if 90 in compute_capabilities and nvcc_cuda_version < Version(\"11.8\"):\n     raise RuntimeError(\n         \"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.\")\n@@ -65,6 +73,7 @@\n     if nvcc_cuda_version >= Version(\"11.1\"):\n         compute_capabilities.add(86)\n     if nvcc_cuda_version >= Version(\"11.8\"):\n+        compute_capabilities.add(89)\n         compute_capabilities.add(90)\n \n # Add target compute capabilities to NVCC flags.\n", "issue": " Issue Building vllm Wheel: Unsupported GPU Architecture 'compute_89'\nI'm trying to build the vllm package from source (pip install vllm), but I'm encountering an error related to CUDA and GPU architecture. I am using RTX 4090. Here are the crucial parts of the log:\r\n\r\n\r\n```\r\nBuilding wheels for collected packages: vllm\r\nBuilding wheel for vllm (pyproject.toml) ... error\r\nerror: subprocess-exited-with-error\r\n\r\n...\r\n\r\nbuilding 'vllm.cache_ops' extension\r\nEmitting ninja build file ...\r\n\r\n...\r\n\r\n/usr/local/cuda/bin/nvcc  -I/tmp/pip-build-env-... (other flags and paths)\r\nnvcc fatal   : Unsupported gpu architecture 'compute_89'\r\n```\r\n\r\nHere is my compute cluster:\r\n---\r\n6790722\r\nAge:15 min.[Remaining](https://vast.ai/faq#Instances):1d 19h71.69.80.64:44536-44750North Carolina, US1x RTX 409081.8  TFLOPS\r\nm:9663host:39244verified0.3/24.6 GB\r\n3506.9 GB/s\r\nROME2D16\r\nPCIE 4.0, 8x12.5 GB/s\r\nAMD EPYC 7662 64-Core Processor\r\n25.6/256 cpu\r\n4/52 GB\r\nnvme\r\n4972 MB/s6.9/21.0 GB\r\n516.4 Mbps514.0 MbpsMax CUDA: 12.2\r\n \r\nGPU: 0% 35C , CPU: 4% Status: success, running pytorch/pytorch_2.0.1-cuda11.7-cudnn8-devel/jupyter\r\n$0.263/hr \r\nCurrent State : running Next State : running\r\nDocker Image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel\r\n\r\nImage Runtype : jupyter_direc ssh_direc ssh_proxy\r\n\r\nEnvironment :\r\n{\r\n  \"JUPYTER_DIR\": \"/\"\r\n}\r\n\r\nDocker Args:\r\n\r\nOnstart: env | grep _ >> /etc/environment; echo 'starting up'\r\n\r\n\r\n---\r\n\r\nIt seems that the nvcc compiler doesn't support the 'compute_89' GPU architecture. I'm using CUDA version 11.7 on a RTX 4090.\r\n\r\nHas anyone encountered this issue before? Any suggestions on how to resolve it?\n", "before_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        f\"Cannot find CUDA_HOME. CUDA must be available in order to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\n# Collect the compute capabilities of all available GPUs.\ndevice_count = torch.cuda.device_count()\ncompute_capabilities: Set[int] = set()\nfor i in range(device_count):\n    major, minor = torch.cuda.get_device_capability(i)\n    if major < 7:\n        raise RuntimeError(\n            \"GPUs with compute capability less than 7.0 are not supported.\")\n    compute_capabilities.add(major * 10 + minor)\n\n# Validate the NVCC CUDA version.\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif 86 in compute_capabilities and nvcc_cuda_version < Version(\"11.1\"):\n    raise RuntimeError(\n        \"CUDA 11.1 or higher is required for GPUs with compute capability 8.6.\")\nif 90 in compute_capabilities and nvcc_cuda_version < Version(\"11.8\"):\n    raise RuntimeError(\n        \"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.\")\n\n# If no GPU is available, add all supported compute capabilities.\nif not compute_capabilities:\n    compute_capabilities = {70, 75, 80}\n    if nvcc_cuda_version >= Version(\"11.1\"):\n        compute_capabilities.add(86)\n    if nvcc_cuda_version >= Version(\"11.8\"):\n        compute_capabilities.add(90)\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{capability},code=sm_{capability}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(activation_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str):\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(\n            r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\", fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file.\"\"\"\n    return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=\"A high-throughput and memory-efficient inference and serving engine for LLMs\",\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(\n        exclude=(\"assets\", \"benchmarks\", \"csrc\", \"docs\", \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n)\n", "path": "setup.py"}], "after_files": [{"content": "import io\nimport os\nimport re\nimport subprocess\nfrom typing import List, Set\n\nfrom packaging.version import parse, Version\nimport setuptools\nimport torch\nfrom torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME\n\nROOT_DIR = os.path.dirname(__file__)\n\n# Compiler flags.\nCXX_FLAGS = [\"-g\", \"-O2\", \"-std=c++17\"]\n# TODO(woosuk): Should we use -O3?\nNVCC_FLAGS = [\"-O2\", \"-std=c++17\"]\n\nABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0\nCXX_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\nNVCC_FLAGS += [f\"-D_GLIBCXX_USE_CXX11_ABI={ABI}\"]\n\nif CUDA_HOME is None:\n    raise RuntimeError(\n        f\"Cannot find CUDA_HOME. CUDA must be available to build the package.\")\n\n\ndef get_nvcc_cuda_version(cuda_dir: str) -> Version:\n    \"\"\"Get the CUDA version from nvcc.\n\n    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py\n    \"\"\"\n    nvcc_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"],\n                                          universal_newlines=True)\n    output = nvcc_output.split()\n    release_idx = output.index(\"release\") + 1\n    nvcc_cuda_version = parse(output[release_idx].split(\",\")[0])\n    return nvcc_cuda_version\n\n\n# Collect the compute capabilities of all available GPUs.\ndevice_count = torch.cuda.device_count()\ncompute_capabilities: Set[int] = set()\nfor i in range(device_count):\n    major, minor = torch.cuda.get_device_capability(i)\n    if major < 7:\n        raise RuntimeError(\n            \"GPUs with compute capability less than 7.0 are not supported.\")\n    compute_capabilities.add(major * 10 + minor)\n\n# Validate the NVCC CUDA version.\nnvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)\nif nvcc_cuda_version < Version(\"11.0\"):\n    raise RuntimeError(\"CUDA 11.0 or higher is required to build the package.\")\nif 86 in compute_capabilities and nvcc_cuda_version < Version(\"11.1\"):\n    raise RuntimeError(\n        \"CUDA 11.1 or higher is required for GPUs with compute capability 8.6.\")\nif 89 in compute_capabilities and nvcc_cuda_version < Version(\"11.8\"):\n    # CUDA 11.8 is required to generate the code targeting compute capability 8.9.\n    # However, GPUs with compute capability 8.9 can also run the code generated by\n    # the previous versions of CUDA 11 and targeting compute capability 8.0.\n    # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0\n    # instead of 8.9.\n    compute_capabilities.remove(89)\n    compute_capabilities.add(80)\nif 90 in compute_capabilities and nvcc_cuda_version < Version(\"11.8\"):\n    raise RuntimeError(\n        \"CUDA 11.8 or higher is required for GPUs with compute capability 9.0.\")\n\n# If no GPU is available, add all supported compute capabilities.\nif not compute_capabilities:\n    compute_capabilities = {70, 75, 80}\n    if nvcc_cuda_version >= Version(\"11.1\"):\n        compute_capabilities.add(86)\n    if nvcc_cuda_version >= Version(\"11.8\"):\n        compute_capabilities.add(89)\n        compute_capabilities.add(90)\n\n# Add target compute capabilities to NVCC flags.\nfor capability in compute_capabilities:\n    NVCC_FLAGS += [\"-gencode\", f\"arch=compute_{capability},code=sm_{capability}\"]\n\n# Use NVCC threads to parallelize the build.\nif nvcc_cuda_version >= Version(\"11.2\"):\n    num_threads = min(os.cpu_count(), 8)\n    NVCC_FLAGS += [\"--threads\", str(num_threads)]\n\next_modules = []\n\n# Cache operations.\ncache_extension = CUDAExtension(\n    name=\"vllm.cache_ops\",\n    sources=[\"csrc/cache.cpp\", \"csrc/cache_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(cache_extension)\n\n# Attention kernels.\nattention_extension = CUDAExtension(\n    name=\"vllm.attention_ops\",\n    sources=[\"csrc/attention.cpp\", \"csrc/attention/attention_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(attention_extension)\n\n# Positional encoding kernels.\npositional_encoding_extension = CUDAExtension(\n    name=\"vllm.pos_encoding_ops\",\n    sources=[\"csrc/pos_encoding.cpp\", \"csrc/pos_encoding_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(positional_encoding_extension)\n\n# Layer normalization kernels.\nlayernorm_extension = CUDAExtension(\n    name=\"vllm.layernorm_ops\",\n    sources=[\"csrc/layernorm.cpp\", \"csrc/layernorm_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(layernorm_extension)\n\n# Activation kernels.\nactivation_extension = CUDAExtension(\n    name=\"vllm.activation_ops\",\n    sources=[\"csrc/activation.cpp\", \"csrc/activation_kernels.cu\"],\n    extra_compile_args={\"cxx\": CXX_FLAGS, \"nvcc\": NVCC_FLAGS},\n)\next_modules.append(activation_extension)\n\n\ndef get_path(*filepath) -> str:\n    return os.path.join(ROOT_DIR, *filepath)\n\n\ndef find_version(filepath: str):\n    \"\"\"Extract version information from the given filepath.\n\n    Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py\n    \"\"\"\n    with open(filepath) as fp:\n        version_match = re.search(\n            r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\", fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\ndef read_readme() -> str:\n    \"\"\"Read the README file.\"\"\"\n    return io.open(get_path(\"README.md\"), \"r\", encoding=\"utf-8\").read()\n\n\ndef get_requirements() -> List[str]:\n    \"\"\"Get Python package dependencies from requirements.txt.\"\"\"\n    with open(get_path(\"requirements.txt\")) as f:\n        requirements = f.read().strip().split(\"\\n\")\n    return requirements\n\n\nsetuptools.setup(\n    name=\"vllm\",\n    version=find_version(get_path(\"vllm\", \"__init__.py\")),\n    author=\"vLLM Team\",\n    license=\"Apache 2.0\",\n    description=\"A high-throughput and memory-efficient inference and serving engine for LLMs\",\n    long_description=read_readme(),\n    long_description_content_type=\"text/markdown\",\n    url=\"https://github.com/vllm-project/vllm\",\n    project_urls={\n        \"Homepage\": \"https://github.com/vllm-project/vllm\",\n        \"Documentation\": \"https://vllm.readthedocs.io/en/latest/\",\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.8\",\n        \"Programming Language :: Python :: 3.9\",\n        \"Programming Language :: Python :: 3.10\",\n        \"License :: OSI Approved :: Apache Software License\",\n        \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    ],\n    packages=setuptools.find_packages(\n        exclude=(\"assets\", \"benchmarks\", \"csrc\", \"docs\", \"examples\", \"tests\")),\n    python_requires=\">=3.8\",\n    install_requires=get_requirements(),\n    ext_modules=ext_modules,\n    cmdclass={\"build_ext\": BuildExtension},\n)\n", "path": "setup.py"}]}