prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  [tensor] fix some unittests
  
  [tensor] fix some unittests
  
  [tensor] fix some unittests
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py`
  Content:
  ```
  1 #!/usr/bin/env python
  2 # -*- encoding: utf-8 -*-
  3 
  4 """
  5 Initialize new tokenizer for continual pre-training
  6 """
  7 
  8 import argparse
  9 import os
  10 import json
  11 from typing import List, Union
  12 
  13 from transformers.models.llama.tokenization_llama import LlamaTokenizer
  14 from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
  15 
  16 from colossalai.logging import get_dist_logger
  17 
  18 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
  19 
  20 logger = get_dist_logger()
  21 
  22 
  23 def expand_vocab_tokenizer(
  24     source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]
  25 ) -> None:
  26     """Expand tokenizer for continue pre-training."""
  27     if os.path.exists(target_tokenizer_dir):
  28         raise RuntimeError(f"Find existed directory {target_tokenizer_dir}")
  29 
  30     source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)
  31     logger.info(source_tokenizer)
  32     source_sp_processor = source_tokenizer.sp_model
  33     source_spm = sp_pb2_model.ModelProto()
  34     source_spm.ParseFromString(source_sp_processor.serialized_model_proto())
  35 
  36     logger.info(f"Source tokenizer size: {len(source_sp_processor)}")
  37 
  38     # Add new tokens to source tokenizer.
  39     source_spm_tokens = set([p.piece for p in source_spm.pieces])
  40     for piece in new_tokens:
  41         assert isinstance(piece, str), f"Invalid token({piece}) type {type(piece)}"
  42         if piece in source_spm_tokens:
  43             # Skip existed token.
  44             continue
  45         new_p = sp_pb2_model.ModelProto().SentencePiece()
  46         new_p.piece = piece
  47         new_p.score = 0
  48         source_spm.pieces.append(new_p)
  49     logger.info(f"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}")
  50 
  51     # Save
  52     os.makedirs(target_tokenizer_dir)
  53     target_tokenizer_model_path = os.path.join(target_tokenizer_dir, "tokenizer.model")
  54     with open(file=target_tokenizer_model_path, mode="wb") as fp:
  55         fp.write(source_spm.SerializeToString())
  56 
  57     target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)
  58     target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)
  59     logger.info(f"Successfully save expand tokenizer to {target_tokenizer_dir}")
  60 
  61 
  62 def main():
  63     parser = argparse.ArgumentParser()
  64     parser.add_argument(
  65         "--source_tokenizer_dir", type=str, required=True, default=None, help="Source tokenizer directory"
  66     )
  67     parser.add_argument(
  68         "--target_tokenizer_dir", type=str, required=True, default=None, help="Target tokenizer directory"
  69     )
  70     parser.add_argument(
  71         "--expand_tokens_file",
  72         type=str,
  73         required=True,
  74         default=None,
  75         help="Path of the file containing tokens to be extended",
  76     )
  77     args = parser.parse_args()
  78 
  79     expand_tokens = []
  80     with open(file=args.expand_tokens_file, mode="r", encoding="utf-8") as fp_reader:
  81         for line in fp_reader:
  82             item = json.loads(line)
  83             # e.g., {"piece": "你好"}
  84             token = item["piece"]
  85             if token in expand_tokens:
  86                 continue
  87             expand_tokens.append(token)
  88     expand_tokens.sort(key=lambda t: len(t), reverse=False)
  89 
  90     expand_vocab_tokenizer(
  91         source_tokenizer_dir=args.source_tokenizer_dir,
  92         target_tokenizer_dir=args.target_tokenizer_dir,
  93         new_tokens=expand_tokens,
  94     )
  95 
  96 
  97 if __name__ == "__main__":
  98     main()
  99
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
  --- a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
  +++ b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
  @@ -6,12 +6,12 @@
   """
   
   import argparse
  -import os
   import json
  +import os
   from typing import List, Union
   
  -from transformers.models.llama.tokenization_llama import LlamaTokenizer
   from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
  +from transformers.models.llama.tokenization_llama import LlamaTokenizer
   
   from colossalai.logging import get_dist_logger
  
verification_info: {"golden_diff": "diff --git a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n--- a/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n+++ b/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py\n@@ -6,12 +6,12 @@\n \"\"\"\n \n import argparse\n-import os\n import json\n+import os\n from typing import List, Union\n \n-from transformers.models.llama.tokenization_llama import LlamaTokenizer\n from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\n+from transformers.models.llama.tokenization_llama import LlamaTokenizer\n \n from colossalai.logging import get_dist_logger\n", "issue": "[tensor] fix some unittests\n\n[tensor] fix some unittests\n\n[tensor] fix some unittests\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\n\"\"\"\nInitialize new tokenizer for continual pre-training\n\"\"\"\n\nimport argparse\nimport os\nimport json\nfrom typing import List, Union\n\nfrom transformers.models.llama.tokenization_llama import LlamaTokenizer\nfrom sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\n\nfrom colossalai.logging import get_dist_logger\n\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\nlogger = get_dist_logger()\n\n\ndef expand_vocab_tokenizer(\n    source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]\n) -> None:\n    \"\"\"Expand tokenizer for continue pre-training.\"\"\"\n    if os.path.exists(target_tokenizer_dir):\n        raise RuntimeError(f\"Find existed directory {target_tokenizer_dir}\")\n\n    source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)\n    logger.info(source_tokenizer)\n    source_sp_processor = source_tokenizer.sp_model\n    source_spm = sp_pb2_model.ModelProto()\n    source_spm.ParseFromString(source_sp_processor.serialized_model_proto())\n\n    logger.info(f\"Source tokenizer size: {len(source_sp_processor)}\")\n\n    # Add new tokens to source tokenizer.\n    source_spm_tokens = set([p.piece for p in source_spm.pieces])\n    for piece in new_tokens:\n        assert isinstance(piece, str), f\"Invalid token({piece}) type {type(piece)}\"\n        if piece in source_spm_tokens:\n            # Skip existed token.\n            continue\n        new_p = sp_pb2_model.ModelProto().SentencePiece()\n        new_p.piece = piece\n        new_p.score = 0\n        source_spm.pieces.append(new_p)\n    logger.info(f\"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}\")\n\n    # Save\n    os.makedirs(target_tokenizer_dir)\n    target_tokenizer_model_path = os.path.join(target_tokenizer_dir, \"tokenizer.model\")\n    with open(file=target_tokenizer_model_path, mode=\"wb\") as fp:\n        fp.write(source_spm.SerializeToString())\n\n    target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)\n    target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)\n    logger.info(f\"Successfully save expand tokenizer to {target_tokenizer_dir}\")\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--source_tokenizer_dir\", type=str, required=True, default=None, help=\"Source tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--target_tokenizer_dir\", type=str, required=True, default=None, help=\"Target tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--expand_tokens_file\",\n        type=str,\n        required=True,\n        default=None,\n        help=\"Path of the file containing tokens to be extended\",\n    )\n    args = parser.parse_args()\n\n    expand_tokens = []\n    with open(file=args.expand_tokens_file, mode=\"r\", encoding=\"utf-8\") as fp_reader:\n        for line in fp_reader:\n            item = json.loads(line)\n            # e.g., {\"piece\": \"\u4f60\u597d\"}\n            token = item[\"piece\"]\n            if token in expand_tokens:\n                continue\n            expand_tokens.append(token)\n    expand_tokens.sort(key=lambda t: len(t), reverse=False)\n\n    expand_vocab_tokenizer(\n        source_tokenizer_dir=args.source_tokenizer_dir,\n        target_tokenizer_dir=args.target_tokenizer_dir,\n        new_tokens=expand_tokens,\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\n\"\"\"\nInitialize new tokenizer for continual pre-training\n\"\"\"\n\nimport argparse\nimport json\nimport os\nfrom typing import List, Union\n\nfrom sentencepiece import sentencepiece_model_pb2 as sp_pb2_model\nfrom transformers.models.llama.tokenization_llama import LlamaTokenizer\n\nfrom colossalai.logging import get_dist_logger\n\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\nlogger = get_dist_logger()\n\n\ndef expand_vocab_tokenizer(\n    source_tokenizer_dir: Union[str, os.PathLike], target_tokenizer_dir: Union[str, os.PathLike], new_tokens: List[str]\n) -> None:\n    \"\"\"Expand tokenizer for continue pre-training.\"\"\"\n    if os.path.exists(target_tokenizer_dir):\n        raise RuntimeError(f\"Find existed directory {target_tokenizer_dir}\")\n\n    source_tokenizer = LlamaTokenizer.from_pretrained(source_tokenizer_dir)\n    logger.info(source_tokenizer)\n    source_sp_processor = source_tokenizer.sp_model\n    source_spm = sp_pb2_model.ModelProto()\n    source_spm.ParseFromString(source_sp_processor.serialized_model_proto())\n\n    logger.info(f\"Source tokenizer size: {len(source_sp_processor)}\")\n\n    # Add new tokens to source tokenizer.\n    source_spm_tokens = set([p.piece for p in source_spm.pieces])\n    for piece in new_tokens:\n        assert isinstance(piece, str), f\"Invalid token({piece}) type {type(piece)}\"\n        if piece in source_spm_tokens:\n            # Skip existed token.\n            continue\n        new_p = sp_pb2_model.ModelProto().SentencePiece()\n        new_p.piece = piece\n        new_p.score = 0\n        source_spm.pieces.append(new_p)\n    logger.info(f\"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}\")\n\n    # Save\n    os.makedirs(target_tokenizer_dir)\n    target_tokenizer_model_path = os.path.join(target_tokenizer_dir, \"tokenizer.model\")\n    with open(file=target_tokenizer_model_path, mode=\"wb\") as fp:\n        fp.write(source_spm.SerializeToString())\n\n    target_tokenizer = LlamaTokenizer(vocab_file=target_tokenizer_model_path)\n    target_tokenizer.save_pretrained(save_directory=target_tokenizer_dir)\n    logger.info(f\"Successfully save expand tokenizer to {target_tokenizer_dir}\")\n\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--source_tokenizer_dir\", type=str, required=True, default=None, help=\"Source tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--target_tokenizer_dir\", type=str, required=True, default=None, help=\"Target tokenizer directory\"\n    )\n    parser.add_argument(\n        \"--expand_tokens_file\",\n        type=str,\n        required=True,\n        default=None,\n        help=\"Path of the file containing tokens to be extended\",\n    )\n    args = parser.parse_args()\n\n    expand_tokens = []\n    with open(file=args.expand_tokens_file, mode=\"r\", encoding=\"utf-8\") as fp_reader:\n        for line in fp_reader:\n            item = json.loads(line)\n            # e.g., {\"piece\": \"\u4f60\u597d\"}\n            token = item[\"piece\"]\n            if token in expand_tokens:\n                continue\n            expand_tokens.append(token)\n    expand_tokens.sort(key=lambda t: len(t), reverse=False)\n\n    expand_vocab_tokenizer(\n        source_tokenizer_dir=args.source_tokenizer_dir,\n        target_tokenizer_dir=args.target_tokenizer_dir,\n        new_tokens=expand_tokens,\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py"}]}