We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[tensor] fix some unittests

[tensor] fix some unittests

[tensor] fix some unittests


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- encoding: utf-8 -*-
3 
4 """
5 Initialize new tokenizer for continual pre-training
6 """
7 
8 import argparse
9 import os
10 import json
11 from typing import List, Union
12 
13 from transformers.models.llama.tokenization_llama import LlamaTokenizer
14 from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
15 
16 from colossalai.logging import get_dist_logger
17 
18 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
19 
20 logger = get_dist_logger()
21 
22 
23 def expand_vocab_tokenizer(
24     source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]
25 ) -> None:
26     """Expand tokenizer for continue pre-training."""
27     if os.path.exists(target_tokenizer_dir):
28         raise RuntimeError(f"Find existed directory {target_tokenizer_dir}")
29 
30     source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)
31     logger.info(source_tokenizer)
32     source_sp_processor = source_tokenizer.sp_model
33     source_spm = sp_pb2_model.ModelProto()
34     source_spm.ParseFromString(source_sp_processor.serialized_model_proto())
35 
36     logger.info(f"Source tokenizer size: {len(source_sp_processor)}")
37 
38     # Add new tokens to source tokenizer.
39     source_spm_tokens = set([p.piece for p in source_spm.pieces])
40     for piece in new_tokens:
41         assert isinstance(piece, str), f"Invalid token({piece}) type {type(piece)}"
42         if piece in source_spm_tokens:
43             # Skip existed token.
44             continue
45         new_p = sp_pb2_model.ModelProto().SentencePiece()
46         new_p.piece = piece
47         new_p.score = 0
48         source_spm.pieces.append(new_p)
49     logger.info(f"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}")
50 
51     # Save
52     os.makedirs(target_tokenizer_dir)
53     target_tokenizer_model_path = os.path.join(target_tokenizer_dir, "tokenizer.model")
54     with open(file=target_tokenizer_model_path, mode="wb") as fp:
55         fp.write(source_spm.SerializeToString())
56 
57     target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)
58     target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)
59     logger.info(f"Successfully save expand tokenizer to {target_tokenizer_dir}")
60 
61 
62 def main():
63     parser = argparse.ArgumentParser()
64     parser.add_argument(
65         "--source_tokenizer_dir", type=str, required=True, default=None, help="Source tokenizer directory"
66     )
67     parser.add_argument(
68         "--target_tokenizer_dir", type=str, required=True, default=None, help="Target tokenizer directory"
69     )
70     parser.add_argument(
71         "--expand_tokens_file",
72         type=str,
73         required=True,
74         default=None,
75         help="Path of the file containing tokens to be extended",
76     )
77     args = parser.parse_args()
78 
79     expand_tokens = []
80     with open(file=args.expand_tokens_file, mode="r", encoding="utf-8") as fp_reader:
81         for line in fp_reader:
82             item = json.loads(line)
83             # e.g., {"piece": "你好"}
84             token = item["piece"]
85             if token in expand_tokens:
86                 continue
87             expand_tokens.append(token)
88     expand_tokens.sort(key=lambda t: len(t), reverse=False)
89 
90     expand_vocab_tokenizer(
91         source_tokenizer_dir=args.source_tokenizer_dir,
92         target_tokenizer_dir=args.target_tokenizer_dir,
93         new_tokens=expand_tokens,
94     )
95 
96 
97 if __name__ == "__main__":
98     main()
99
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
--- a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
+++ b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
@@ -6,12 +6,12 @@
 """
 
 import argparse
-import os
 import json
+import os
 from typing import List, Union
 
-from transformers.models.llama.tokenization_llama import LlamaTokenizer
 from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
+from transformers.models.llama.tokenization_llama import LlamaTokenizer
 
 from colossalai.logging import get_dist_logger

{"golden_diff": "diff --git a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n--- a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n+++ b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n@@ -6,12 +6,12 @@\n \"\"\"\n \n import argparse\n-import os\n import json\n+import os\n from typing import List, Union\n \n-from transformers.models.llama.tokenization_llama import LlamaTokenizer\n from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\n+from transformers.models.llama.tokenization_llama import LlamaTokenizer\n \n from colossalai.logging import get_dist_logger\n", "issue": "[tensor] fix some unittests\n\n[tensor] fix some unittests\n\n[tensor] fix some unittests\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\n\"\"\"\nInitialize new tokenizer for continual pre-training\n\"\"\"\n\nimport argparse\nimport os\nimport json\nfrom typing import List, Union\n\nfrom transformers.models.llama.tokenization_llama import LlamaTokenizer\nfrom sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\n\nfrom colossalai.logging import get_dist_logger\n\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\nlogger = get_dist_logger()\n\n\ndef expand_vocab_tokenizer(\n    source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]\n) -> None:\n    \"\"\"Expand tokenizer for continue pre-training.\"\"\"\n    if os.path.exists(target_tokenizer_dir):\n        raise RuntimeError(f\"Find existed directory {target_tokenizer_dir}\")\n\n    source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)\n    logger.info(source_tokenizer)\n    source_sp_processor = source_tokenizer.sp_model\n    source_spm = sp_pb2_model.ModelProto()\n    source_spm.ParseFromString(source_sp_processor.serialized_model_proto())\n\n    logger.info(f\"Source tokenizer size: {len(source_sp_processor)}\")\n\n    # Add new tokens to source tokenizer.\n    source_spm_tokens = set([p.piece for p in source_spm.pieces])\n    for piece in new_tokens:\n        assert isinstance(piece, str), f\"Invalid token({piece}) type {type(piece)}\"\n        if piece in source_spm_tokens:\n            # Skip existed token.\n            continue\n        new_p = sp_pb2_model.ModelProto().SentencePiece()\n        new_p.piece = piece\n        new_p.score = 0\n        source_spm.pieces.append(new_p)\n    logger.info(f\"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}\")\n\n    # Save\n    os.makedirs(target_tokenizer_dir)\n    target_tokenizer_model_path = os.path.join(target_tokenizer_dir, \"tokenizer.model\")\n    with open(file=target_tokenizer_model_path, mode=\"wb\") as fp:\n        fp.write(source_spm.SerializeToString())\n\n    target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)\n    target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)\n    logger.info(f\"Successfully save expand tokenizer to {target_tokenizer_dir}\")\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--source_tokenizer_dir\", type=str, required=True, default=None, help=\"Source tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--target_tokenizer_dir\", type=str, required=True, default=None, help=\"Target tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--expand_tokens_file\",\n        type=str,\n        required=True,\n        default=None,\n        help=\"Path of the file containing tokens to be extended\",\n    )\n    args = parser.parse_args()\n\n    expand_tokens = []\n    with open(file=args.expand_tokens_file, mode=\"r\", encoding=\"utf-8\") as fp_reader:\n        for line in fp_reader:\n            item = json.loads(line)\n            # e.g., {\"piece\": \"\u4f60\u597d\"}\n            token = item[\"piece\"]\n            if token in expand_tokens:\n                continue\n            expand_tokens.append(token)\n    expand_tokens.sort(key=lambda t: len(t), reverse=False)\n\n    expand_vocab_tokenizer(\n        source_tokenizer_dir=args.source_tokenizer_dir,\n        target_tokenizer_dir=args.target_tokenizer_dir,\n        new_tokens=expand_tokens,\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\n\"\"\"\nInitialize new tokenizer for continual pre-training\n\"\"\"\n\nimport argparse\nimport json\nimport os\nfrom typing import List, Union\n\nfrom sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\nfrom transformers.models.llama.tokenization_llama import LlamaTokenizer\n\nfrom colossalai.logging import get_dist_logger\n\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\nlogger = get_dist_logger()\n\n\ndef expand_vocab_tokenizer(\n    source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]\n) -> None:\n    \"\"\"Expand tokenizer for continue pre-training.\"\"\"\n    if os.path.exists(target_tokenizer_dir):\n        raise RuntimeError(f\"Find existed directory {target_tokenizer_dir}\")\n\n    source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)\n    logger.info(source_tokenizer)\n    source_sp_processor = source_tokenizer.sp_model\n    source_spm = sp_pb2_model.ModelProto()\n    source_spm.ParseFromString(source_sp_processor.serialized_model_proto())\n\n    logger.info(f\"Source tokenizer size: {len(source_sp_processor)}\")\n\n    # Add new tokens to source tokenizer.\n    source_spm_tokens = set([p.piece for p in source_spm.pieces])\n    for piece in new_tokens:\n        assert isinstance(piece, str), f\"Invalid token({piece}) type {type(piece)}\"\n        if piece in source_spm_tokens:\n            # Skip existed token.\n            continue\n        new_p = sp_pb2_model.ModelProto().SentencePiece()\n        new_p.piece = piece\n        new_p.score = 0\n        source_spm.pieces.append(new_p)\n    logger.info(f\"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}\")\n\n    # Save\n    os.makedirs(target_tokenizer_dir)\n    target_tokenizer_model_path = os.path.join(target_tokenizer_dir, \"tokenizer.model\")\n    with open(file=target_tokenizer_model_path, mode=\"wb\") as fp:\n        fp.write(source_spm.SerializeToString())\n\n    target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)\n    target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)\n    logger.info(f\"Successfully save expand tokenizer to {target_tokenizer_dir}\")\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--source_tokenizer_dir\", type=str, required=True, default=None, help=\"Source tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--target_tokenizer_dir\", type=str, required=True, default=None, help=\"Target tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--expand_tokens_file\",\n        type=str,\n        required=True,\n        default=None,\n        help=\"Path of the file containing tokens to be extended\",\n    )\n    args = parser.parse_args()\n\n    expand_tokens = []\n    with open(file=args.expand_tokens_file, mode=\"r\", encoding=\"utf-8\") as fp_reader:\n        for line in fp_reader:\n            item = json.loads(line)\n            # e.g., {\"piece\": \"\u4f60\u597d\"}\n            token = item[\"piece\"]\n            if token in expand_tokens:\n                continue\n            expand_tokens.append(token)\n    expand_tokens.sort(key=lambda t: len(t), reverse=False)\n\n    expand_vocab_tokenizer(\n        source_tokenizer_dir=args.source_tokenizer_dir,\n        target_tokenizer_dir=args.target_tokenizer_dir,\n        new_tokens=expand_tokens,\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py"}]}