prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  [BUG]: 运行chatgpt推理示例报错
  ### 🐛 Describe the bug
  
  (https://github.com/hpcaitech/ColossalAI/tree/main/applications/ChatGPT)/examples/    运行inference.py 抛出OSError:
  ![image](https://user-images.githubusercontent.com/51046864/222064682-05cc668a-aee6-4f05-a2d7-3379169a1835.png)
  
  
  ### Environment
  
  _No response_
  [tensor] fix some unittests
  
  [tensor] fix some unittests
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `applications/ChatGPT/examples/inference.py`
  Content:
  ```
  1 import argparse
  2 import torch
  3 
  4 from chatgpt.nn import BLOOMActor, GPTActor, OPTActor
  5 from transformers import AutoTokenizer
  6 from transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer
  7 
  8 
  9 def eval(args):
  10     # configure model
  11     if args.model == 'gpt2':
  12         model = GPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  13     elif args.model == 'bloom':
  14         model = BLOOMActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  15     elif args.model == 'opt':
  16         model = OPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  17     else:
  18         raise ValueError(f'Unsupported model "{args.model}"')
  19 
  20     # configure tokenizer
  21     if args.model == 'gpt2':
  22         tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
  23         tokenizer.pad_token = tokenizer.eos_token
  24     elif args.model == 'bloom':
  25         tokenizer = AutoTokenizer.from_pretrained(args.pretrain)
  26         tokenizer.pad_token = tokenizer.eos_token
  27     elif args.model == 'opt':
  28         tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
  29     else:
  30         raise ValueError(f'Unsupported model "{args.model}"')
  31 
  32     model.eval()
  33     input = args.input
  34     input_ids = tokenizer.encode(input, return_tensors='pt').to(torch.cuda.current_device())
  35     outputs = model.generate(input_ids,
  36                              max_length=args.max_length,
  37                              do_sample=True,
  38                              top_k=50,
  39                              top_p=0.95,
  40                              num_return_sequences=1)
  41     output = tokenizer.batch_decode(outputs[0], skip_special_tokens=True)
  42     print(output)
  43 
  44 
  45 if __name__ == '__main__':
  46     parser = argparse.ArgumentParser()
  47     parser.add_argument('--model', default='gpt2', choices=['gpt2', 'bloom', 'opt'])
  48     parser.add_argument('--pretrain', type=str, default=None)
  49     parser.add_argument('--input', type=str, default='Q: How are you ? A:')
  50     parser.add_argument('--max_length', type=int, default=100)
  51     args = parser.parse_args()
  52     eval(args)
  53
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/applications/ChatGPT/examples/inference.py b/applications/ChatGPT/examples/inference.py
  --- a/applications/ChatGPT/examples/inference.py
  +++ b/applications/ChatGPT/examples/inference.py
  @@ -9,30 +9,34 @@
   def eval(args):
       # configure model
       if args.model == 'gpt2':
  -        model = GPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  +        actor = GPTActor().to(torch.cuda.current_device())
       elif args.model == 'bloom':
  -        model = BLOOMActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  +        actor = BLOOMActor().to(torch.cuda.current_device())
       elif args.model == 'opt':
  -        model = OPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())
  +        actor = OPTActor().to(torch.cuda.current_device())
       else:
           raise ValueError(f'Unsupported model "{args.model}"')
   
  +    state_dict = torch.load(args.pretrain)
  +    actor.model.load_state_dict(state_dict)
  +    
  +    
       # configure tokenizer
       if args.model == 'gpt2':
           tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
           tokenizer.pad_token = tokenizer.eos_token
       elif args.model == 'bloom':
  -        tokenizer = AutoTokenizer.from_pretrained(args.pretrain)
  +        tokenizer = AutoTokenizer.from_pretrained('bigscience/bloom-560m')
           tokenizer.pad_token = tokenizer.eos_token
       elif args.model == 'opt':
  -        tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m")
  +        tokenizer = AutoTokenizer.from_pretrained('facebook/opt-350m')
       else:
           raise ValueError(f'Unsupported model "{args.model}"')
   
  -    model.eval()
  +    actor.eval()
       input = args.input
       input_ids = tokenizer.encode(input, return_tensors='pt').to(torch.cuda.current_device())
  -    outputs = model.generate(input_ids,
  +    outputs = actor.generate(input_ids,
                                max_length=args.max_length,
                                do_sample=True,
                                top_k=50,
  @@ -46,7 +50,7 @@
       parser = argparse.ArgumentParser()
       parser.add_argument('--model', default='gpt2', choices=['gpt2', 'bloom', 'opt'])
       parser.add_argument('--pretrain', type=str, default=None)
  -    parser.add_argument('--input', type=str, default='Q: How are you ? A:')
  +    parser.add_argument('--input', type=str, default='Question: How are you ? Answer:')
       parser.add_argument('--max_length', type=int, default=100)
       args = parser.parse_args()
       eval(args)
  
verification_info: {"golden_diff": "diff --git a/applications/ChatGPT/examples/inference.py b/applications/ChatGPT/examples/inference.py\n--- a/applications/ChatGPT/examples/inference.py\n+++ b/applications/ChatGPT/examples/inference.py\n@@ -9,30 +9,34 @@\n def eval(args):\n     # configure model\n     if args.model == 'gpt2':\n-        model = GPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n+        actor = GPTActor().to(torch.cuda.current_device())\n     elif args.model == 'bloom':\n-        model = BLOOMActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n+        actor = BLOOMActor().to(torch.cuda.current_device())\n     elif args.model == 'opt':\n-        model = OPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n+        actor = OPTActor().to(torch.cuda.current_device())\n     else:\n         raise ValueError(f'Unsupported model \"{args.model}\"')\n \n+    state_dict = torch.load(args.pretrain)\n+    actor.model.load_state_dict(state_dict)\n+    \n+    \n     # configure tokenizer\n     if args.model == 'gpt2':\n         tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n         tokenizer.pad_token = tokenizer.eos_token\n     elif args.model == 'bloom':\n-        tokenizer = AutoTokenizer.from_pretrained(args.pretrain)\n+        tokenizer = AutoTokenizer.from_pretrained('bigscience/bloom-560m')\n         tokenizer.pad_token = tokenizer.eos_token\n     elif args.model == 'opt':\n-        tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n+        tokenizer = AutoTokenizer.from_pretrained('facebook/opt-350m')\n     else:\n         raise ValueError(f'Unsupported model \"{args.model}\"')\n \n-    model.eval()\n+    actor.eval()\n     input = args.input\n     input_ids = tokenizer.encode(input, return_tensors='pt').to(torch.cuda.current_device())\n-    outputs = model.generate(input_ids,\n+    outputs = actor.generate(input_ids,\n                              max_length=args.max_length,\n                              do_sample=True,\n                              top_k=50,\n@@ -46,7 +50,7 @@\n     parser = argparse.ArgumentParser()\n     parser.add_argument('--model', default='gpt2', choices=['gpt2', 'bloom', 'opt'])\n     parser.add_argument('--pretrain', type=str, default=None)\n-    parser.add_argument('--input', type=str, default='Q: How are you ? A:')\n+    parser.add_argument('--input', type=str, default='Question: How are you ? Answer:')\n     parser.add_argument('--max_length', type=int, default=100)\n     args = parser.parse_args()\n     eval(args)\n", "issue": "[BUG]: \u8fd0\u884cchatgpt\u63a8\u7406\u793a\u4f8b\u62a5\u9519\n### \ud83d\udc1b Describe the bug\n\n(https://github.com/hpcaitech/ColossalAI/tree/main/applications/ChatGPT)/examples/    \u8fd0\u884cinference.py \u629b\u51faOSError:\r\n![image](https://user-images.githubusercontent.com/51046864/222064682-05cc668a-aee6-4f05-a2d7-3379169a1835.png)\r\n\n\n### Environment\n\n_No response_\n[tensor] fix some unittests\n\n[tensor] fix some unittests\n\n", "before_files": [{"content": "import argparse\nimport torch\n\nfrom chatgpt.nn import BLOOMActor, GPTActor, OPTActor\nfrom transformers import AutoTokenizer\nfrom transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer\n\n\ndef eval(args):\n    # configure model\n    if args.model == 'gpt2':\n        model = GPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n    elif args.model == 'bloom':\n        model = BLOOMActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n    elif args.model == 'opt':\n        model = OPTActor(pretrained=args.pretrain).to(torch.cuda.current_device())\n    else:\n        raise ValueError(f'Unsupported model \"{args.model}\"')\n\n    # configure tokenizer\n    if args.model == 'gpt2':\n        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n        tokenizer.pad_token = tokenizer.eos_token\n    elif args.model == 'bloom':\n        tokenizer = AutoTokenizer.from_pretrained(args.pretrain)\n        tokenizer.pad_token = tokenizer.eos_token\n    elif args.model == 'opt':\n        tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n    else:\n        raise ValueError(f'Unsupported model \"{args.model}\"')\n\n    model.eval()\n    input = args.input\n    input_ids = tokenizer.encode(input, return_tensors='pt').to(torch.cuda.current_device())\n    outputs = model.generate(input_ids,\n                             max_length=args.max_length,\n                             do_sample=True,\n                             top_k=50,\n                             top_p=0.95,\n                             num_return_sequences=1)\n    output = tokenizer.batch_decode(outputs[0], skip_special_tokens=True)\n    print(output)\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--model', default='gpt2', choices=['gpt2', 'bloom', 'opt'])\n    parser.add_argument('--pretrain', type=str, default=None)\n    parser.add_argument('--input', type=str, default='Q: How are you ? A:')\n    parser.add_argument('--max_length', type=int, default=100)\n    args = parser.parse_args()\n    eval(args)\n", "path": "applications/ChatGPT/examples/inference.py"}], "after_files": [{"content": "import argparse\nimport torch\n\nfrom chatgpt.nn import BLOOMActor, GPTActor, OPTActor\nfrom transformers import AutoTokenizer\nfrom transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer\n\n\ndef eval(args):\n    # configure model\n    if args.model == 'gpt2':\n        actor = GPTActor().to(torch.cuda.current_device())\n    elif args.model == 'bloom':\n        actor = BLOOMActor().to(torch.cuda.current_device())\n    elif args.model == 'opt':\n        actor = OPTActor().to(torch.cuda.current_device())\n    else:\n        raise ValueError(f'Unsupported model \"{args.model}\"')\n\n    state_dict = torch.load(args.pretrain)\n    actor.model.load_state_dict(state_dict)\n    \n    \n    # configure tokenizer\n    if args.model == 'gpt2':\n        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n        tokenizer.pad_token = tokenizer.eos_token\n    elif args.model == 'bloom':\n        tokenizer = AutoTokenizer.from_pretrained('bigscience/bloom-560m')\n        tokenizer.pad_token = tokenizer.eos_token\n    elif args.model == 'opt':\n        tokenizer = AutoTokenizer.from_pretrained('facebook/opt-350m')\n    else:\n        raise ValueError(f'Unsupported model \"{args.model}\"')\n\n    actor.eval()\n    input = args.input\n    input_ids = tokenizer.encode(input, return_tensors='pt').to(torch.cuda.current_device())\n    outputs = actor.generate(input_ids,\n                             max_length=args.max_length,\n                             do_sample=True,\n                             top_k=50,\n                             top_p=0.95,\n                             num_return_sequences=1)\n    output = tokenizer.batch_decode(outputs[0], skip_special_tokens=True)\n    print(output)\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--model', default='gpt2', choices=['gpt2', 'bloom', 'opt'])\n    parser.add_argument('--pretrain', type=str, default=None)\n    parser.add_argument('--input', type=str, default='Question: How are you ? Answer:')\n    parser.add_argument('--max_length', type=int, default=100)\n    args = parser.parse_args()\n    eval(args)\n", "path": "applications/ChatGPT/examples/inference.py"}]}