We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[Feature] Make torchvision optional
### Motivation

Since v0.4.2, `torchvision` has been introduced into the LMDeploy runtime because it is used in VLMs. 
https://github.com/InternLM/lmdeploy/blob/54b7230b4ca08b37b85e5f6e1960e2445dca5288/requirements/runtime.txt#L16-L18
For some users, like me who might only want to use LLM, theoretically I have no dependency on `torchvision`. But currently, due to the Python code relying on `torchvision`, even if I only use LLM inference, I am still required to install `torchvision`. 
https://github.com/InternLM/lmdeploy/blob/aa8f7d17ec2abc968241cad32070b27eb315d472/lmdeploy/vl/model/cogvlm.py#L6-L8
https://github.com/InternLM/lmdeploy/blob/aa8f7d17ec2abc968241cad32070b27eb315d472/lmdeploy/vl/model/internvl.py#L112-L127
This is not very user-friendly as **the dependencies increase** and this particular **dependency also has a large binary size**. Could you consider making it optional? Thanks. @lvhan028 @irexyc @grimoire @AllentDan 

### Related resources

_No response_

### Additional context

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lmdeploy/archs.py`
Content:
```
1 # Copyright (c) OpenMMLab. All rights reserved.
2 import os
3 from typing import Literal, Optional, Union
4 
5 from lmdeploy.serve.async_engine import AsyncEngine
6 from lmdeploy.serve.vl_async_engine import VLAsyncEngine
7 from lmdeploy.utils import get_hf_config_content
8 
9 from .messages import PytorchEngineConfig, TurbomindEngineConfig
10 from .utils import get_logger
11 
12 SUPPORTED_TASKS = {'llm': AsyncEngine, 'vlm': VLAsyncEngine}
13 
14 logger = get_logger('lmdeploy')
15 
16 
17 def autoget_backend(model_path: str) -> Literal['turbomind', 'pytorch']:
18     """Get backend type in auto backend mode.
19 
20     Args:
21          model_path (str): the path of a model.
22             It could be one of the following options:
23                 - i) A local directory path of a turbomind model which is
24                     converted by `lmdeploy convert` command or download from
25                     ii) and iii).
26                 - ii) The model_id of a lmdeploy-quantized model hosted
27                     inside a model repo on huggingface.co, such as
28                     "InternLM/internlm-chat-20b-4bit",
29                     "lmdeploy/llama2-chat-70b-4bit", etc.
30                 - iii) The model_id of a model hosted inside a model repo
31                     on huggingface.co, such as "internlm/internlm-chat-7b",
32                     "Qwen/Qwen-7B-Chat ", "baichuan-inc/Baichuan2-7B-Chat"
33                     and so on.
34 
35     Returns:
36         str: the backend type.
37     """
38     from lmdeploy.pytorch.supported_models import \
39         is_supported as is_supported_pytorch
40 
41     pytorch_has, turbomind_has = False, False
42     is_turbomind_installed = True
43     try:
44         from lmdeploy.turbomind.supported_models import \
45             is_supported as is_supported_turbomind
46         turbomind_has = is_supported_turbomind(model_path)
47     except ImportError:
48         is_turbomind_installed = False
49 
50     pytorch_has = is_supported_pytorch(model_path)
51 
52     try_run_msg = (f'Try to run with pytorch engine because `{model_path}`'
53                    ' is not explicitly supported by lmdeploy. ')
54     if is_turbomind_installed:
55         if not turbomind_has:
56             if pytorch_has:
57                 logger.warning('Fallback to pytorch engine because '
58                                f'`{model_path}` not supported by turbomind'
59                                ' engine.')
60             else:
61                 logger.warning(try_run_msg)
62     else:
63         logger.warning(
64             'Fallback to pytorch engine because turbomind engine is not '
65             'installed correctly. If you insist to use turbomind engine, '
66             'you may need to reinstall lmdeploy from pypi or build from '
67             'source and try again.')
68         if not pytorch_has:
69             logger.warning(try_run_msg)
70 
71     backend = 'turbomind' if turbomind_has else 'pytorch'
72     return backend
73 
74 
75 def autoget_backend_config(
76     model_path: str,
77     backend_config: Optional[Union[PytorchEngineConfig,
78                                    TurbomindEngineConfig]] = None
79 ) -> Union[PytorchEngineConfig, TurbomindEngineConfig]:
80     """Get backend config automatically.
81 
82     Args:
83         model_path (str): The input model path.
84         backend_config (TurbomindEngineConfig | PytorchEngineConfig): The
85             input backend config. Default to None.
86 
87     Returns:
88         (PytorchEngineConfig | TurbomindEngineConfig): The auto-determined
89             backend engine config.
90     """
91     from dataclasses import asdict
92 
93     backend = autoget_backend(model_path)
94     if backend == 'pytorch':
95         config = PytorchEngineConfig()
96     else:
97         config = TurbomindEngineConfig()
98     if backend_config is not None:
99         if type(backend_config) == type(config):
100             return backend_config
101         else:
102             data = asdict(backend_config)
103             for k, v in data.items():
104                 if v and hasattr(config, k):
105                     setattr(config, k, v)
106             # map attributes with different names
107             if type(backend_config) is TurbomindEngineConfig:
108                 config.block_size = backend_config.cache_block_seq_len
109             else:
110                 config.cache_block_seq_len = backend_config.block_size
111     return config
112 
113 
114 def check_vl_llm(config: dict) -> bool:
115     """check if the model is a vl model from model config."""
116     if 'auto_map' in config:
117         for _, v in config['auto_map'].items():
118             if 'InternLMXComposer2ForCausalLM' in v:
119                 return True
120     arch = config['architectures'][0]
121     if arch in ['LlavaLlamaForCausalLM', 'LlavaMistralForCausalLM']:
122         return True
123     elif arch == 'QWenLMHeadModel' and 'visual' in config:
124         return True
125     elif arch == 'MultiModalityCausalLM' and 'language_config' in config:
126         return True
127     elif arch == 'CogVLMForCausalLM':
128         return True
129     elif arch == 'InternLMXComposer2ForCausalLM':
130         return True
131     elif arch == 'InternVLChatModel':
132         return True
133     elif arch in ['MiniGeminiLlamaForCausalLM', 'MGMLlamaForCausalLM']:
134         return True
135     return False
136 
137 
138 def get_task(model_path: str):
139     """get pipeline type and pipeline class from model config."""
140     if os.path.exists(os.path.join(model_path, 'triton_models', 'weights')):
141         # workspace model
142         return 'llm', AsyncEngine
143     config = get_hf_config_content(model_path)
144     if check_vl_llm(config):
145         return 'vlm', VLAsyncEngine
146 
147     # default task, pipeline_class
148     return 'llm', AsyncEngine
149
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lmdeploy/archs.py b/lmdeploy/archs.py
--- a/lmdeploy/archs.py
+++ b/lmdeploy/archs.py
@@ -2,15 +2,11 @@
 import os
 from typing import Literal, Optional, Union
 
-from lmdeploy.serve.async_engine import AsyncEngine
-from lmdeploy.serve.vl_async_engine import VLAsyncEngine
 from lmdeploy.utils import get_hf_config_content
 
 from .messages import PytorchEngineConfig, TurbomindEngineConfig
 from .utils import get_logger
 
-SUPPORTED_TASKS = {'llm': AsyncEngine, 'vlm': VLAsyncEngine}
-
 logger = get_logger('lmdeploy')
 
 
@@ -137,11 +133,14 @@
 
 def get_task(model_path: str):
     """get pipeline type and pipeline class from model config."""
+    from lmdeploy.serve.async_engine import AsyncEngine
+
     if os.path.exists(os.path.join(model_path, 'triton_models', 'weights')):
         # workspace model
         return 'llm', AsyncEngine
     config = get_hf_config_content(model_path)
     if check_vl_llm(config):
+        from lmdeploy.serve.vl_async_engine import VLAsyncEngine
         return 'vlm', VLAsyncEngine
 
     # default task, pipeline_class

{"golden_diff": "diff --git a/lmdeploy/archs.py b/lmdeploy/archs.py\n--- a/lmdeploy/archs.py\n+++ b/lmdeploy/archs.py\n@@ -2,15 +2,11 @@\n import os\n from typing import Literal, Optional, Union\n \n-from lmdeploy.serve.async_engine import AsyncEngine\n-from lmdeploy.serve.vl_async_engine import VLAsyncEngine\n from lmdeploy.utils import get_hf_config_content\n \n from .messages import PytorchEngineConfig, TurbomindEngineConfig\n from .utils import get_logger\n \n-SUPPORTED_TASKS = {'llm': AsyncEngine, 'vlm': VLAsyncEngine}\n-\n logger = get_logger('lmdeploy')\n \n \n@@ -137,11 +133,14 @@\n \n def get_task(model_path: str):\n     \"\"\"get pipeline type and pipeline class from model config.\"\"\"\n+    from lmdeploy.serve.async_engine import AsyncEngine\n+\n     if os.path.exists(os.path.join(model_path, 'triton_models', 'weights')):\n         # workspace model\n         return 'llm', AsyncEngine\n     config = get_hf_config_content(model_path)\n     if check_vl_llm(config):\n+        from lmdeploy.serve.vl_async_engine import VLAsyncEngine\n         return 'vlm', VLAsyncEngine\n \n     # default task, pipeline_class\n", "issue": "[Feature] Make torchvision optional\n### Motivation\n\nSince v0.4.2, `torchvision` has been introduced into the LMDeploy runtime because it is used in VLMs. \r\nhttps://github.com/InternLM/lmdeploy/blob/54b7230b4ca08b37b85e5f6e1960e2445dca5288/requirements/runtime.txt#L16-L18\r\nFor some users, like me who might only want to use LLM, theoretically I have no dependency on `torchvision`. But currently, due to the Python code relying on `torchvision`, even if I only use LLM inference, I am still required to install `torchvision`. \r\nhttps://github.com/InternLM/lmdeploy/blob/aa8f7d17ec2abc968241cad32070b27eb315d472/lmdeploy/vl/model/cogvlm.py#L6-L8\r\nhttps://github.com/InternLM/lmdeploy/blob/aa8f7d17ec2abc968241cad32070b27eb315d472/lmdeploy/vl/model/internvl.py#L112-L127\r\nThis is not very user-friendly as **the dependencies increase** and this particular **dependency also has a large binary size**. Could you consider making it optional? Thanks. @lvhan028 @irexyc @grimoire @AllentDan \n\n### Related resources\n\n_No response_\n\n### Additional context\n\n_No response_\n", "before_files": [{"content": "# Copyright (c) OpenMMLab. All rights reserved.\nimport os\nfrom typing import Literal, Optional, Union\n\nfrom lmdeploy.serve.async_engine import AsyncEngine\nfrom lmdeploy.serve.vl_async_engine import VLAsyncEngine\nfrom lmdeploy.utils import get_hf_config_content\n\nfrom .messages import PytorchEngineConfig, TurbomindEngineConfig\nfrom .utils import get_logger\n\nSUPPORTED_TASKS = {'llm': AsyncEngine, 'vlm': VLAsyncEngine}\n\nlogger = get_logger('lmdeploy')\n\n\ndef autoget_backend(model_path: str) -> Literal['turbomind', 'pytorch']:\n    \"\"\"Get backend type in auto backend mode.\n\n    Args:\n         model_path (str): the path of a model.\n            It could be one of the following options:\n                - i) A local directory path of a turbomind model which is\n                    converted by `lmdeploy convert` command or download from\n                    ii) and iii).\n                - ii) The model_id of a lmdeploy-quantized model hosted\n                    inside a model repo on huggingface.co, such as\n                    \"InternLM/internlm-chat-20b-4bit\",\n                    \"lmdeploy/llama2-chat-70b-4bit\", etc.\n                - iii) The model_id of a model hosted inside a model repo\n                    on huggingface.co, such as \"internlm/internlm-chat-7b\",\n                    \"Qwen/Qwen-7B-Chat \", \"baichuan-inc/Baichuan2-7B-Chat\"\n                    and so on.\n\n    Returns:\n        str: the backend type.\n    \"\"\"\n    from lmdeploy.pytorch.supported_models import \\\n        is_supported as is_supported_pytorch\n\n    pytorch_has, turbomind_has = False, False\n    is_turbomind_installed = True\n    try:\n        from lmdeploy.turbomind.supported_models import \\\n            is_supported as is_supported_turbomind\n        turbomind_has = is_supported_turbomind(model_path)\n    except ImportError:\n        is_turbomind_installed = False\n\n    pytorch_has = is_supported_pytorch(model_path)\n\n    try_run_msg = (f'Try to run with pytorch engine because `{model_path}`'\n                   ' is not explicitly supported by lmdeploy. ')\n    if is_turbomind_installed:\n        if not turbomind_has:\n            if pytorch_has:\n                logger.warning('Fallback to pytorch engine because '\n                               f'`{model_path}` not supported by turbomind'\n                               ' engine.')\n            else:\n                logger.warning(try_run_msg)\n    else:\n        logger.warning(\n            'Fallback to pytorch engine because turbomind engine is not '\n            'installed correctly. If you insist to use turbomind engine, '\n            'you may need to reinstall lmdeploy from pypi or build from '\n            'source and try again.')\n        if not pytorch_has:\n            logger.warning(try_run_msg)\n\n    backend = 'turbomind' if turbomind_has else 'pytorch'\n    return backend\n\n\ndef autoget_backend_config(\n    model_path: str,\n    backend_config: Optional[Union[PytorchEngineConfig,\n                                   TurbomindEngineConfig]] = None\n) -> Union[PytorchEngineConfig, TurbomindEngineConfig]:\n    \"\"\"Get backend config automatically.\n\n    Args:\n        model_path (str): The input model path.\n        backend_config (TurbomindEngineConfig | PytorchEngineConfig): The\n            input backend config. Default to None.\n\n    Returns:\n        (PytorchEngineConfig | TurbomindEngineConfig): The auto-determined\n            backend engine config.\n    \"\"\"\n    from dataclasses import asdict\n\n    backend = autoget_backend(model_path)\n    if backend == 'pytorch':\n        config = PytorchEngineConfig()\n    else:\n        config = TurbomindEngineConfig()\n    if backend_config is not None:\n        if type(backend_config) == type(config):\n            return backend_config\n        else:\n            data = asdict(backend_config)\n            for k, v in data.items():\n                if v and hasattr(config, k):\n                    setattr(config, k, v)\n            # map attributes with different names\n            if type(backend_config) is TurbomindEngineConfig:\n                config.block_size = backend_config.cache_block_seq_len\n            else:\n                config.cache_block_seq_len = backend_config.block_size\n    return config\n\n\ndef check_vl_llm(config: dict) -> bool:\n    \"\"\"check if the model is a vl model from model config.\"\"\"\n    if 'auto_map' in config:\n        for _, v in config['auto_map'].items():\n            if 'InternLMXComposer2ForCausalLM' in v:\n                return True\n    arch = config['architectures'][0]\n    if arch in ['LlavaLlamaForCausalLM', 'LlavaMistralForCausalLM']:\n        return True\n    elif arch == 'QWenLMHeadModel' and 'visual' in config:\n        return True\n    elif arch == 'MultiModalityCausalLM' and 'language_config' in config:\n        return True\n    elif arch == 'CogVLMForCausalLM':\n        return True\n    elif arch == 'InternLMXComposer2ForCausalLM':\n        return True\n    elif arch == 'InternVLChatModel':\n        return True\n    elif arch in ['MiniGeminiLlamaForCausalLM', 'MGMLlamaForCausalLM']:\n        return True\n    return False\n\n\ndef get_task(model_path: str):\n    \"\"\"get pipeline type and pipeline class from model config.\"\"\"\n    if os.path.exists(os.path.join(model_path, 'triton_models', 'weights')):\n        # workspace model\n        return 'llm', AsyncEngine\n    config = get_hf_config_content(model_path)\n    if check_vl_llm(config):\n        return 'vlm', VLAsyncEngine\n\n    # default task, pipeline_class\n    return 'llm', AsyncEngine\n", "path": "lmdeploy/archs.py"}], "after_files": [{"content": "# Copyright (c) OpenMMLab. All rights reserved.\nimport os\nfrom typing import Literal, Optional, Union\n\nfrom lmdeploy.utils import get_hf_config_content\n\nfrom .messages import PytorchEngineConfig, TurbomindEngineConfig\nfrom .utils import get_logger\n\nlogger = get_logger('lmdeploy')\n\n\ndef autoget_backend(model_path: str) -> Literal['turbomind', 'pytorch']:\n    \"\"\"Get backend type in auto backend mode.\n\n    Args:\n         model_path (str): the path of a model.\n            It could be one of the following options:\n                - i) A local directory path of a turbomind model which is\n                    converted by `lmdeploy convert` command or download from\n                    ii) and iii).\n                - ii) The model_id of a lmdeploy-quantized model hosted\n                    inside a model repo on huggingface.co, such as\n                    \"InternLM/internlm-chat-20b-4bit\",\n                    \"lmdeploy/llama2-chat-70b-4bit\", etc.\n                - iii) The model_id of a model hosted inside a model repo\n                    on huggingface.co, such as \"internlm/internlm-chat-7b\",\n                    \"Qwen/Qwen-7B-Chat \", \"baichuan-inc/Baichuan2-7B-Chat\"\n                    and so on.\n\n    Returns:\n        str: the backend type.\n    \"\"\"\n    from lmdeploy.pytorch.supported_models import \\\n        is_supported as is_supported_pytorch\n\n    pytorch_has, turbomind_has = False, False\n    is_turbomind_installed = True\n    try:\n        from lmdeploy.turbomind.supported_models import \\\n            is_supported as is_supported_turbomind\n        turbomind_has = is_supported_turbomind(model_path)\n    except ImportError:\n        is_turbomind_installed = False\n\n    pytorch_has = is_supported_pytorch(model_path)\n\n    try_run_msg = (f'Try to run with pytorch engine because `{model_path}`'\n                   ' is not explicitly supported by lmdeploy. ')\n    if is_turbomind_installed:\n        if not turbomind_has:\n            if pytorch_has:\n                logger.warning('Fallback to pytorch engine because '\n                               f'`{model_path}` not supported by turbomind'\n                               ' engine.')\n            else:\n                logger.warning(try_run_msg)\n    else:\n        logger.warning(\n            'Fallback to pytorch engine because turbomind engine is not '\n            'installed correctly. If you insist to use turbomind engine, '\n            'you may need to reinstall lmdeploy from pypi or build from '\n            'source and try again.')\n        if not pytorch_has:\n            logger.warning(try_run_msg)\n\n    backend = 'turbomind' if turbomind_has else 'pytorch'\n    return backend\n\n\ndef autoget_backend_config(\n    model_path: str,\n    backend_config: Optional[Union[PytorchEngineConfig,\n                                   TurbomindEngineConfig]] = None\n) -> Union[PytorchEngineConfig, TurbomindEngineConfig]:\n    \"\"\"Get backend config automatically.\n\n    Args:\n        model_path (str): The input model path.\n        backend_config (TurbomindEngineConfig | PytorchEngineConfig): The\n            input backend config. Default to None.\n\n    Returns:\n        (PytorchEngineConfig | TurbomindEngineConfig): The auto-determined\n            backend engine config.\n    \"\"\"\n    from dataclasses import asdict\n\n    backend = autoget_backend(model_path)\n    if backend == 'pytorch':\n        config = PytorchEngineConfig()\n    else:\n        config = TurbomindEngineConfig()\n    if backend_config is not None:\n        if type(backend_config) == type(config):\n            return backend_config\n        else:\n            data = asdict(backend_config)\n            for k, v in data.items():\n                if v and hasattr(config, k):\n                    setattr(config, k, v)\n            # map attributes with different names\n            if type(backend_config) is TurbomindEngineConfig:\n                config.block_size = backend_config.cache_block_seq_len\n            else:\n                config.cache_block_seq_len = backend_config.block_size\n    return config\n\n\ndef check_vl_llm(config: dict) -> bool:\n    \"\"\"check if the model is a vl model from model config.\"\"\"\n    if 'auto_map' in config:\n        for _, v in config['auto_map'].items():\n            if 'InternLMXComposer2ForCausalLM' in v:\n                return True\n    arch = config['architectures'][0]\n    if arch in ['LlavaLlamaForCausalLM', 'LlavaMistralForCausalLM']:\n        return True\n    elif arch == 'QWenLMHeadModel' and 'visual' in config:\n        return True\n    elif arch == 'MultiModalityCausalLM' and 'language_config' in config:\n        return True\n    elif arch == 'CogVLMForCausalLM':\n        return True\n    elif arch == 'InternLMXComposer2ForCausalLM':\n        return True\n    elif arch == 'InternVLChatModel':\n        return True\n    elif arch in ['MiniGeminiLlamaForCausalLM', 'MGMLlamaForCausalLM']:\n        return True\n    return False\n\n\ndef get_task(model_path: str):\n    \"\"\"get pipeline type and pipeline class from model config.\"\"\"\n    from lmdeploy.serve.async_engine import AsyncEngine\n\n    if os.path.exists(os.path.join(model_path, 'triton_models', 'weights')):\n        # workspace model\n        return 'llm', AsyncEngine\n    config = get_hf_config_content(model_path)\n    if check_vl_llm(config):\n        from lmdeploy.serve.vl_async_engine import VLAsyncEngine\n        return 'vlm', VLAsyncEngine\n\n    # default task, pipeline_class\n    return 'llm', AsyncEngine\n", "path": "lmdeploy/archs.py"}]}