prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  BertGenerationTokenizer provides an unexpected value for BertGenerationModel
  - `transformers` version: 4.2.2
  - PyTorch version (GPU?): 1.7.0+cu101
  - tokenizers: @n1t0, @LysandreJik
  
  ## Information
  
  in both models BertGenerationEncoder,  BertGenerationDecoder, there's no need for `token_type_ids` however the BertGenerationTokenizer provides it, this issue will be raised if you want to input the tokenizer results directly with `**`,
  
  and if it meant to be like this, and the user should be aware of this behaviour, I think a change should be in the documentation.
  
  Note: Another issue with BertGenerationTokenizer is the necessity of sentencepiece module, do you prefer that it should for the user to install it separately or it should be included in transformers dependencies.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `src/transformers/models/bert_generation/tokenization_bert_generation.py`
  Content:
  ```
  1 # coding=utf-8
  2 # Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
  3 #
  4 # Licensed under the Apache License, Version 2.0 (the "License");
  5 # you may not use this file except in compliance with the License.
  6 # You may obtain a copy of the License at
  7 #
  8 #     http://www.apache.org/licenses/LICENSE-2.0
  9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """ Tokenization class for model BertGeneration."""
  16 
  17 
  18 import os
  19 from shutil import copyfile
  20 from typing import List, Optional, Tuple
  21 
  22 import sentencepiece as spm
  23 
  24 from ...tokenization_utils import PreTrainedTokenizer
  25 from ...utils import logging
  26 
  27 
  28 logger = logging.get_logger(__name__)
  29 
  30 VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}
  31 
  32 tokenizer_url = "https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model"
  33 
  34 
  35 class BertGenerationTokenizer(PreTrainedTokenizer):
  36     """
  37     Construct a BertGeneration tokenizer. Based on `SentencePiece <https://github.com/google/sentencepiece>`__.
  38 
  39     This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods.
  40     Users should refer to this superclass for more information regarding those methods.
  41 
  42     Args:
  43         vocab_file (:obj:`str`):
  44             `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that
  45             contains the vocabulary necessary to instantiate a tokenizer.
  46         eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
  47             The end of sequence token.
  48         bos_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
  49             The begin of sequence token.
  50         unk_token (:obj:`str`, `optional`, defaults to :obj:`"<unk>"`):
  51             The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
  52             token instead.
  53         pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
  54             The token used for padding, for example when batching sequences of different lengths.
  55     """
  56 
  57     vocab_files_names = VOCAB_FILES_NAMES
  58     pretrained_vocab_files_map = {"vocab_file": {"bert_for_seq_generation": tokenizer_url}}
  59     max_model_input_sizes = {"bert_for_seq_generation": 512}
  60     prefix_tokens: List[int] = []
  61 
  62     def __init__(
  63         self,
  64         vocab_file,
  65         bos_token="<s>",
  66         eos_token="</s>",
  67         unk_token="<unk>",
  68         pad_token="<pad>",
  69         sep_token="<::::>",
  70         **kwargs
  71     ):
  72         # Add extra_ids to the special token list
  73         super().__init__(
  74             bos_token=bos_token,
  75             eos_token=eos_token,
  76             unk_token=unk_token,
  77             pad_token=pad_token,
  78             sep_token=sep_token,
  79             **kwargs,
  80         )
  81 
  82         self.vocab_file = vocab_file
  83 
  84         self.sp_model = spm.SentencePieceProcessor()
  85         self.sp_model.Load(vocab_file)
  86 
  87     @property
  88     def vocab_size(self):
  89         return self.sp_model.get_piece_size()
  90 
  91     def get_vocab(self):
  92         vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
  93         vocab.update(self.added_tokens_encoder)
  94         return vocab
  95 
  96     def __getstate__(self):
  97         state = self.__dict__.copy()
  98         state["sp_model"] = None
  99         return state
  100 
  101     def __setstate__(self, d):
  102         self.__dict__ = d
  103         self.sp_model = spm.SentencePieceProcessor()
  104         self.sp_model.Load(self.vocab_file)
  105 
  106     def _tokenize(self, text, sample=False):
  107         """Take as input a string and return a list of strings (tokens) for words/sub-words"""
  108         if not sample:
  109             pieces = self.sp_model.EncodeAsPieces(text)
  110         else:
  111             pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1)
  112         return pieces
  113 
  114     def _convert_token_to_id(self, token):
  115         """ Converts a token (str) in an id using the vocab. """
  116         return self.sp_model.piece_to_id(token)
  117 
  118     def _convert_id_to_token(self, index):
  119         """Converts an index (integer) in a token (str) using the vocab."""
  120         token = self.sp_model.IdToPiece(index)
  121         return token
  122 
  123     def convert_tokens_to_string(self, tokens):
  124         """ Converts a sequence of tokens (string) in a single string. """
  125         out_string = self.sp_model.decode_pieces(tokens)
  126         return out_string
  127 
  128     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
  129         if not os.path.isdir(save_directory):
  130             logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
  131             return
  132         out_vocab_file = os.path.join(
  133             save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
  134         )
  135 
  136         if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
  137             copyfile(self.vocab_file, out_vocab_file)
  138 
  139         return (out_vocab_file,)
  140
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/src/transformers/models/bert_generation/tokenization_bert_generation.py b/src/transformers/models/bert_generation/tokenization_bert_generation.py
  --- a/src/transformers/models/bert_generation/tokenization_bert_generation.py
  +++ b/src/transformers/models/bert_generation/tokenization_bert_generation.py
  @@ -58,6 +58,7 @@
       pretrained_vocab_files_map = {"vocab_file": {"bert_for_seq_generation": tokenizer_url}}
       max_model_input_sizes = {"bert_for_seq_generation": 512}
       prefix_tokens: List[int] = []
  +    model_input_names = ["input_ids", "attention_mask"]
   
       def __init__(
           self,
  
verification_info: {"golden_diff": "diff --git a/src/transformers/models/bert_generation/tokenization_bert_generation.py b/src/transformers/models/bert_generation/tokenization_bert_generation.py\n--- a/src/transformers/models/bert_generation/tokenization_bert_generation.py\n+++ b/src/transformers/models/bert_generation/tokenization_bert_generation.py\n@@ -58,6 +58,7 @@\n     pretrained_vocab_files_map = {\"vocab_file\": {\"bert_for_seq_generation\": tokenizer_url}}\n     max_model_input_sizes = {\"bert_for_seq_generation\": 512}\n     prefix_tokens: List[int] = []\n+    model_input_names = [\"input_ids\", \"attention_mask\"]\n \n     def __init__(\n         self,\n", "issue": "BertGenerationTokenizer provides an unexpected value for BertGenerationModel\n- `transformers` version: 4.2.2\r\n- PyTorch version (GPU?): 1.7.0+cu101\r\n- tokenizers: @n1t0, @LysandreJik\r\n\r\n## Information\r\n\r\nin both models BertGenerationEncoder,  BertGenerationDecoder, there's no need for `token_type_ids` however the BertGenerationTokenizer provides it, this issue will be raised if you want to input the tokenizer results directly with `**`,\r\n\r\nand if it meant to be like this, and the user should be aware of this behaviour, I think a change should be in the documentation.\r\n\r\nNote: Another issue with BertGenerationTokenizer is the necessity of sentencepiece module, do you prefer that it should for the user to install it separately or it should be included in transformers dependencies.\r\n\n", "before_files": [{"content": "# coding=utf-8\n# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\" Tokenization class for model BertGeneration.\"\"\"\n\n\nimport os\nfrom shutil import copyfile\nfrom typing import List, Optional, Tuple\n\nimport sentencepiece as spm\n\nfrom ...tokenization_utils import PreTrainedTokenizer\nfrom ...utils import logging\n\n\nlogger = logging.get_logger(__name__)\n\nVOCAB_FILES_NAMES = {\"vocab_file\": \"spiece.model\"}\n\ntokenizer_url = \"https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model\"\n\n\nclass BertGenerationTokenizer(PreTrainedTokenizer):\n    \"\"\"\n    Construct a BertGeneration tokenizer. Based on `SentencePiece <https://github.com/google/sentencepiece>`__.\n\n    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods.\n    Users should refer to this superclass for more information regarding those methods.\n\n    Args:\n        vocab_file (:obj:`str`):\n            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that\n            contains the vocabulary necessary to instantiate a tokenizer.\n        eos_token (:obj:`str`, `optional`, defaults to :obj:`\"</s>\"`):\n            The end of sequence token.\n        bos_token (:obj:`str`, `optional`, defaults to :obj:`\"<s>\"`):\n            The begin of sequence token.\n        unk_token (:obj:`str`, `optional`, defaults to :obj:`\"<unk>\"`):\n            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this\n            token instead.\n        pad_token (:obj:`str`, `optional`, defaults to :obj:`\"<pad>\"`):\n            The token used for padding, for example when batching sequences of different lengths.\n    \"\"\"\n\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = {\"vocab_file\": {\"bert_for_seq_generation\": tokenizer_url}}\n    max_model_input_sizes = {\"bert_for_seq_generation\": 512}\n    prefix_tokens: List[int] = []\n\n    def __init__(\n        self,\n        vocab_file,\n        bos_token=\"<s>\",\n        eos_token=\"</s>\",\n        unk_token=\"<unk>\",\n        pad_token=\"<pad>\",\n        sep_token=\"<::::>\",\n        **kwargs\n    ):\n        # Add extra_ids to the special token list\n        super().__init__(\n            bos_token=bos_token,\n            eos_token=eos_token,\n            unk_token=unk_token,\n            pad_token=pad_token,\n            sep_token=sep_token,\n            **kwargs,\n        )\n\n        self.vocab_file = vocab_file\n\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(vocab_file)\n\n    @property\n    def vocab_size(self):\n        return self.sp_model.get_piece_size()\n\n    def get_vocab(self):\n        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}\n        vocab.update(self.added_tokens_encoder)\n        return vocab\n\n    def __getstate__(self):\n        state = self.__dict__.copy()\n        state[\"sp_model\"] = None\n        return state\n\n    def __setstate__(self, d):\n        self.__dict__ = d\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(self.vocab_file)\n\n    def _tokenize(self, text, sample=False):\n        \"\"\"Take as input a string and return a list of strings (tokens) for words/sub-words\"\"\"\n        if not sample:\n            pieces = self.sp_model.EncodeAsPieces(text)\n        else:\n            pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1)\n        return pieces\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n        return self.sp_model.piece_to_id(token)\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n        token = self.sp_model.IdToPiece(index)\n        return token\n\n    def convert_tokens_to_string(self, tokens):\n        \"\"\" Converts a sequence of tokens (string) in a single string. \"\"\"\n        out_string = self.sp_model.decode_pieces(tokens)\n        return out_string\n\n    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        out_vocab_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"vocab_file\"]\n        )\n\n        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):\n            copyfile(self.vocab_file, out_vocab_file)\n\n        return (out_vocab_file,)\n", "path": "src/transformers/models/bert_generation/tokenization_bert_generation.py"}], "after_files": [{"content": "# coding=utf-8\n# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\" Tokenization class for model BertGeneration.\"\"\"\n\n\nimport os\nfrom shutil import copyfile\nfrom typing import List, Optional, Tuple\n\nimport sentencepiece as spm\n\nfrom ...tokenization_utils import PreTrainedTokenizer\nfrom ...utils import logging\n\n\nlogger = logging.get_logger(__name__)\n\nVOCAB_FILES_NAMES = {\"vocab_file\": \"spiece.model\"}\n\ntokenizer_url = \"https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model\"\n\n\nclass BertGenerationTokenizer(PreTrainedTokenizer):\n    \"\"\"\n    Construct a BertGeneration tokenizer. Based on `SentencePiece <https://github.com/google/sentencepiece>`__.\n\n    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods.\n    Users should refer to this superclass for more information regarding those methods.\n\n    Args:\n        vocab_file (:obj:`str`):\n            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that\n            contains the vocabulary necessary to instantiate a tokenizer.\n        eos_token (:obj:`str`, `optional`, defaults to :obj:`\"</s>\"`):\n            The end of sequence token.\n        bos_token (:obj:`str`, `optional`, defaults to :obj:`\"<s>\"`):\n            The begin of sequence token.\n        unk_token (:obj:`str`, `optional`, defaults to :obj:`\"<unk>\"`):\n            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this\n            token instead.\n        pad_token (:obj:`str`, `optional`, defaults to :obj:`\"<pad>\"`):\n            The token used for padding, for example when batching sequences of different lengths.\n    \"\"\"\n\n    vocab_files_names = VOCAB_FILES_NAMES\n    pretrained_vocab_files_map = {\"vocab_file\": {\"bert_for_seq_generation\": tokenizer_url}}\n    max_model_input_sizes = {\"bert_for_seq_generation\": 512}\n    prefix_tokens: List[int] = []\n    model_input_names = [\"input_ids\", \"attention_mask\"]\n\n    def __init__(\n        self,\n        vocab_file,\n        bos_token=\"<s>\",\n        eos_token=\"</s>\",\n        unk_token=\"<unk>\",\n        pad_token=\"<pad>\",\n        sep_token=\"<::::>\",\n        **kwargs\n    ):\n        # Add extra_ids to the special token list\n        super().__init__(\n            bos_token=bos_token,\n            eos_token=eos_token,\n            unk_token=unk_token,\n            pad_token=pad_token,\n            sep_token=sep_token,\n            **kwargs,\n        )\n\n        self.vocab_file = vocab_file\n\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(vocab_file)\n\n    @property\n    def vocab_size(self):\n        return self.sp_model.get_piece_size()\n\n    def get_vocab(self):\n        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}\n        vocab.update(self.added_tokens_encoder)\n        return vocab\n\n    def __getstate__(self):\n        state = self.__dict__.copy()\n        state[\"sp_model\"] = None\n        return state\n\n    def __setstate__(self, d):\n        self.__dict__ = d\n        self.sp_model = spm.SentencePieceProcessor()\n        self.sp_model.Load(self.vocab_file)\n\n    def _tokenize(self, text, sample=False):\n        \"\"\"Take as input a string and return a list of strings (tokens) for words/sub-words\"\"\"\n        if not sample:\n            pieces = self.sp_model.EncodeAsPieces(text)\n        else:\n            pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1)\n        return pieces\n\n    def _convert_token_to_id(self, token):\n        \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n        return self.sp_model.piece_to_id(token)\n\n    def _convert_id_to_token(self, index):\n        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n        token = self.sp_model.IdToPiece(index)\n        return token\n\n    def convert_tokens_to_string(self, tokens):\n        \"\"\" Converts a sequence of tokens (string) in a single string. \"\"\"\n        out_string = self.sp_model.decode_pieces(tokens)\n        return out_string\n\n    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        out_vocab_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"vocab_file\"]\n        )\n\n        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):\n            copyfile(self.vocab_file, out_vocab_file)\n\n        return (out_vocab_file,)\n", "path": "src/transformers/models/bert_generation/tokenization_bert_generation.py"}]}