We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BlenderbotSmallTokenizer throws tuple index out of range error for stopword
Using transformers==3.4.0

Script used:
```

from transformers import BlenderbotSmallTokenizer, BlenderbotForConditionalGeneration
mname = 'facebook/blenderbot-90M'
tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)

sentence = "."
tokenizer(sentence)['input_ids']
```

This throws `IndexError: tuple index out of range`

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/transformers/tokenization_blenderbot.py`
Content:
```
1 #!/usr/bin/env python3
2 # coding=utf-8
3 # Copyright (c) Facebook, Inc. and its affiliates.
4 #
5 # This source code is licensed under the MIT license found in the;
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 # LICENSE file in the root directory of this source tree.
17 """"BlenderbotTokenizer and BlenderbotSmallTokenizer"""
18 import json
19 import os
20 from typing import Dict, List, Optional, Tuple
21 
22 import regex as re
23 
24 from .tokenization_roberta import RobertaTokenizer
25 from .tokenization_utils import PreTrainedTokenizer
26 from .utils import logging
27 
28 
29 logger = logging.get_logger(__name__)
30 
31 
32 VOCAB_FILES_NAMES = {
33     "vocab_file": "vocab.json",
34     "merges_file": "merges.txt",
35     # "tokenizer_config_file": "tokenizer_config.json",
36 }
37 CKPT_3B = "facebook/blenderbot-3B"
38 
39 
40 class BlenderbotTokenizer(RobertaTokenizer):
41     r"""
42     Construct a Blenderbot tokenizer.
43 
44     :class:`~transformers.Blenderbot` is nearly identical to :class:`~transformers.RobertaTokenizer` and runs
45     end-to-end tokenization: punctuation splitting and wordpiece. The only difference is that it doesnt add BOS
46     token to the beginning of sequences.
47 
48     Refer to superclass :class:`~transformers.RobertaTokenizer` for usage examples and documentation concerning
49     parameters.
50     """
51     vocab_files_names = {
52         "vocab_file": "vocab.json",
53         "merges_file": "merges.txt",
54         "tokenizer_config_file": "tokenizer_config.json",
55     }
56     pretrained_vocab_files_map = {
57         "vocab_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json"},
58         "merges_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt"},
59         "tokenizer_config_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json"},
60     }
61     max_model_input_sizes = {"facebook/blenderbot-3B": 128}
62 
63     def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: List[int] = None):
64         """
65         Build model inputs from a sequence or a pair of sequence for sequence classification tasks
66         by concatenating and adding special tokens.
67         A Blenderbot sequence has the following format:
68 
69         - single sequence: `` X </s>``
70 
71         Args:
72             token_ids_0 (:obj:`List[int]`):
73                 List of IDs to which the special tokens will be added
74             token_ids_1 (:obj:`List[int]`, `optional`):
75                 Will be ignored
76 
77         Returns:
78             :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
79         """
80         return token_ids_0 + [self.eos_token_id]
81 
82 
83 def get_pairs(word):
84     """Return set of symbol pairs in a word.
85 
86     Word is represented as tuple of symbols (symbols being variable-length strings).
87     """
88     pairs = set()
89     prev_char = word[0]
90     for char in word[1:]:
91         pairs.add((prev_char, char))
92         prev_char = char
93 
94     pairs = set(pairs)
95     return pairs
96 
97 
98 class BlenderbotSmallTokenizer(PreTrainedTokenizer):
99     """
100     Constructs a Blenderbot-90M tokenizer based on BPE (Byte-Pair-Encoding)
101 
102     This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods. Users
103     should refer to the superclass for more information regarding methods.
104 
105     Args:
106         vocab_file (:obj:`str`):
107             File containing the vocabulary.
108         merges_file (:obj:`str`):
109             Path to the merges file.
110         bos_token (:obj:`str`, `optional`, defaults to :obj:`"__start__"`):
111             The beginning of sentence token.
112         eos_token (:obj:`str`, `optional`, defaults to :obj:`"__end__"`):
113             The end of sentence token.
114         unk_token (:obj:`str`, `optional`, defaults to :obj:`"__unk__"`):
115             The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead.
116         pad_token (:obj:`str`, `optional`, defaults to :obj:`"__pad__"`):
117             The token used for padding, for example when batching sequences of different lengths.
118         **kwargs
119             Additional keyword arguments passed along to  :class:`~transformers.PreTrainedTokenizer`
120     """
121 
122     vocab_files_names = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
123     pretrained_vocab_files_map = {
124         "vocab_file": {"facebook/blenderbot-90M": "https://cdn.huggingface.co/facebook/blenderbot-90M/vocab.json"},
125         "merges_file": {"facebook/blenderbot-90M": "https://cdn.huggingface.co/facebook/blenderbot-90M/merges.txt"},
126     }
127     max_model_input_sizes = {"facebook/blenderbot-90M": 512}
128 
129     def __init__(
130         self,
131         vocab_file,
132         merges_file,
133         bos_token="__start__",
134         eos_token="__end__",
135         unk_token="__unk__",
136         pad_token="__null",
137         **kwargs
138     ):
139         super().__init__(unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, pad_token=pad_token, **kwargs)
140 
141         with open(vocab_file, encoding="utf-8") as vocab_handle:
142             self.encoder = json.load(vocab_handle)
143         self.decoder = {v: k for k, v in self.encoder.items()}
144         with open(merges_file, encoding="utf-8") as merges_handle:
145             merges = merges_handle.read().split("\n")[1:-1]
146         merges = [tuple(merge.split()) for merge in merges]
147         self.bpe_ranks = dict(zip(merges, range(len(merges))))
148         self.cache = {}
149 
150     @property
151     def vocab_size(self) -> int:
152         return len(self.encoder)
153 
154     def get_vocab(self) -> Dict:
155         return dict(self.encoder, **self.added_tokens_encoder)
156 
157     def bpe(self, token: str) -> str:
158         if token in self.cache:
159             return self.cache[token]
160         token = re.sub("([.,!?()])", r" \1", token)
161         token = re.sub("(')", r" \1 ", token)
162         token = re.sub("\s{2,}", " ", token)
163         if "\n" in token:
164             token = token.replace("\n", " __newln__")
165 
166         tokens = token.split(" ")
167         words = []
168         for token in tokens:
169             token = token.lower()
170             word = tuple(token)
171             word = tuple(list(word[:-1]) + [word[-1] + "</w>"])
172             pairs = get_pairs(word)
173 
174             if not pairs:
175                 words.append(token)
176                 continue
177 
178             while True:
179                 bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
180                 if bigram not in self.bpe_ranks:
181                     break
182                 first, second = bigram
183                 new_word = []
184                 i = 0
185 
186                 while i < len(word):
187                     try:
188                         j = word.index(first, i)
189                         new_word.extend(word[i:j])
190                         i = j
191                     except ValueError:
192                         new_word.extend(word[i:])
193                         break
194 
195                     if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
196                         new_word.append(first + second)
197                         i += 2
198                     else:
199                         new_word.append(word[i])
200                         i += 1
201                 new_word = tuple(new_word)
202                 word = new_word
203                 if len(word) == 1:
204                     break
205                 else:
206                     pairs = get_pairs(word)
207             word = "@@ ".join(word)
208             word = word[:-4]
209 
210             self.cache[token] = word
211             words.append(word)
212         return " ".join(words)
213 
214     def _tokenize(self, text: str) -> List[str]:
215         """ Split a string into tokens using BPE."""
216         split_tokens = []
217 
218         words = re.findall(r"\S+\n?", text)
219 
220         for token in words:
221             split_tokens.extend([t for t in self.bpe(token).split(" ")])
222         return split_tokens
223 
224     def _convert_token_to_id(self, token: str) -> int:
225         """ Converts a token to an id using the vocab. """
226         token = token.lower()
227         return self.encoder.get(token, self.encoder.get(self.unk_token))
228 
229     def _convert_id_to_token(self, index: int) -> str:
230         """Converts an index (integer) in a token (str) using the vocab."""
231         return self.decoder.get(index, self.unk_token)
232 
233     def convert_tokens_to_string(self, tokens: List[str]) -> str:
234         """ Converts a sequence of tokens  in a single string. """
235         out_string = " ".join(tokens).replace("@@ ", "").strip()
236         return out_string
237 
238     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
239         if not os.path.isdir(save_directory):
240             logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
241             return
242         vocab_file = os.path.join(
243             save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
244         )
245         merge_file = os.path.join(
246             save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
247         )
248 
249         with open(vocab_file, "w", encoding="utf-8") as f:
250             f.write(json.dumps(self.encoder, ensure_ascii=False))
251 
252         index = 0
253         with open(merge_file, "w", encoding="utf-8") as writer:
254             writer.write("#version: 0.2\n")
255             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
256                 if index != token_index:
257                     logger.warning(
258                         "Saving vocabulary to {}: BPE merge indices are not consecutive."
259                         " Please check that the tokenizer is not corrupted!".format(merge_file)
260                     )
261                     index = token_index
262                 writer.write(" ".join(bpe_tokens) + "\n")
263                 index += 1
264 
265         return vocab_file, merge_file
266
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/transformers/tokenization_blenderbot.py b/src/transformers/tokenization_blenderbot.py
--- a/src/transformers/tokenization_blenderbot.py
+++ b/src/transformers/tokenization_blenderbot.py
@@ -166,6 +166,9 @@
         tokens = token.split(" ")
         words = []
         for token in tokens:
+            if not len(token):
+                continue
+
             token = token.lower()
             word = tuple(token)
             word = tuple(list(word[:-1]) + [word[-1] + "</w>"])

{"golden_diff": "diff --git a/src/transformers/tokenization_blenderbot.py b/src/transformers/tokenization_blenderbot.py\n--- a/src/transformers/tokenization_blenderbot.py\n+++ b/src/transformers/tokenization_blenderbot.py\n@@ -166,6 +166,9 @@\n         tokens = token.split(\" \")\n         words = []\n         for token in tokens:\n+            if not len(token):\n+                continue\n+\n             token = token.lower()\n             word = tuple(token)\n             word = tuple(list(word[:-1]) + [word[-1] + \"</w>\"])\n", "issue": "BlenderbotSmallTokenizer throws tuple index out of range error for stopword\nUsing transformers==3.4.0\r\n\r\nScript used:\r\n```\r\n\r\nfrom transformers import BlenderbotSmallTokenizer, BlenderbotForConditionalGeneration\r\nmname = 'facebook/blenderbot-90M'\r\ntokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)\r\n\r\nsentence = \".\"\r\ntokenizer(sentence)['input_ids']\r\n```\r\n\r\nThis throws `IndexError: tuple index out of range`\n", "before_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\n# Copyright (c) Facebook, Inc. and its affiliates.\n#\n# This source code is licensed under the MIT license found in the;\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# LICENSE file in the root directory of this source tree.\n\"\"\"\"BlenderbotTokenizer and BlenderbotSmallTokenizer\"\"\"\nimport json\nimport os\nfrom typing import Dict, List, Optional, Tuple\n\nimport regex as re\n\nfrom .tokenization_roberta import RobertaTokenizer\nfrom .tokenization_utils import PreTrainedTokenizer\nfrom .utils import logging\n\n\nlogger = logging.get_logger(__name__)\n\n\nVOCAB_FILES_NAMES = {\n    \"vocab_file\": \"vocab.json\",\n    \"merges_file\": \"merges.txt\",\n    # \"tokenizer_config_file\": \"tokenizer_config.json\",\n}\nCKPT_3B = \"facebook/blenderbot-3B\"\n\n\nclass BlenderbotTokenizer(RobertaTokenizer):\n    r\"\"\"\n    Construct a Blenderbot tokenizer.\n\n    :class:`~transformers.Blenderbot` is nearly identical to :class:`~transformers.RobertaTokenizer` and runs\n    end-to-end tokenization: punctuation splitting and wordpiece. The only difference is that it doesnt add BOS\n    token to the beginning of sequences.\n\n    Refer to superclass :class:`~transformers.RobertaTokenizer` for usage examples and documentation concerning\n    parameters.\n    \"\"\"\n    vocab_files_names = {\n        \"vocab_file\": \"vocab.json\",\n        \"merges_file\": \"merges.txt\",\n        \"tokenizer_config_file\": \"tokenizer_config.json\",\n    }\n    pretrained_vocab_files_map = {\n        \"vocab_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json\"},\n        \"merges_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt\"},\n        \"tokenizer_config_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json\"},\n    }\n    max_model_input_sizes = {\"facebook/blenderbot-3B\": 128}\n\n    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: List[int] = None):\n        \"\"\"\n        Build model inputs from a sequence or a pair of sequence for sequence classification tasks\n        by concatenating and adding special tokens.\n        A Blenderbot sequence has the following format:\n\n        - single sequence: `` X </s>``\n\n        Args:\n            token_ids_0 (:obj:`List[int]`):\n                List of IDs to which the special tokens will be added\n            token_ids_1 (:obj:`List[int]`, `optional`):\n                Will be ignored\n\n        Returns:\n            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.\n        \"\"\"\n        return token_ids_0 + [self.eos_token_id]\n\n\ndef get_pairs(word):\n    \"\"\"Return set of symbol pairs in a word.\n\n    Word is represented as tuple of symbols (symbols being variable-length strings).\n    \"\"\"\n    pairs = set()\n    prev_char = word[0]\n    for char in word[1:]:\n        pairs.add((prev_char, char))\n        prev_char = char\n\n    pairs = set(pairs)\n    return pairs\n\n\nclass BlenderbotSmallTokenizer(PreTrainedTokenizer):\n    \"\"\"\n    Constructs a Blenderbot-90M tokenizer based on BPE (Byte-Pair-Encoding)\n\n    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods. Users\n    should refer to the superclass for more information regarding methods.\n\n    Args:\n        vocab_file (:obj:`str`):\n            File containing the vocabulary.\n        merges_file (:obj:`str`):\n            Path to the merges file.\n        bos_token (:obj:`str`, `optional`, defaults to :obj:`\"__start__\"`):\n            The beginning of sentence token.\n        eos_token (:obj:`str`, `optional`, defaults to :obj:`\"__end__\"`):\n            The end of sentence token.\n        unk_token (:obj:`str`, `optional`, defaults to :obj:`\"__unk__\"`):\n            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead.\n        pad_token (:obj:`str`, `optional`, defaults to :obj:`\"__pad__\"`):\n            The token used for padding, for example when batching sequences of different lengths.\n        **kwargs\n            Additional keyword arguments passed along to  :class:`~transformers.PreTrainedTokenizer`\n    \"\"\"\n\n    vocab_files_names = {\"vocab_file\": \"vocab.json\", \"merges_file\": \"merges.txt\"}\n    pretrained_vocab_files_map = {\n        \"vocab_file\": {\"facebook/blenderbot-90M\": \"https://cdn.huggingface.co/facebook/blenderbot-90M/vocab.json\"},\n        \"merges_file\": {\"facebook/blenderbot-90M\": \"https://cdn.huggingface.co/facebook/blenderbot-90M/merges.txt\"},\n    }\n    max_model_input_sizes = {\"facebook/blenderbot-90M\": 512}\n\n    def __init__(\n        self,\n        vocab_file,\n        merges_file,\n        bos_token=\"__start__\",\n        eos_token=\"__end__\",\n        unk_token=\"__unk__\",\n        pad_token=\"__null\",\n        **kwargs\n    ):\n        super().__init__(unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, pad_token=pad_token, **kwargs)\n\n        with open(vocab_file, encoding=\"utf-8\") as vocab_handle:\n            self.encoder = json.load(vocab_handle)\n        self.decoder = {v: k for k, v in self.encoder.items()}\n        with open(merges_file, encoding=\"utf-8\") as merges_handle:\n            merges = merges_handle.read().split(\"\\n\")[1:-1]\n        merges = [tuple(merge.split()) for merge in merges]\n        self.bpe_ranks = dict(zip(merges, range(len(merges))))\n        self.cache = {}\n\n    @property\n    def vocab_size(self) -> int:\n        return len(self.encoder)\n\n    def get_vocab(self) -> Dict:\n        return dict(self.encoder, **self.added_tokens_encoder)\n\n    def bpe(self, token: str) -> str:\n        if token in self.cache:\n            return self.cache[token]\n        token = re.sub(\"([.,!?()])\", r\" \\1\", token)\n        token = re.sub(\"(')\", r\" \\1 \", token)\n        token = re.sub(\"\\s{2,}\", \" \", token)\n        if \"\\n\" in token:\n            token = token.replace(\"\\n\", \" __newln__\")\n\n        tokens = token.split(\" \")\n        words = []\n        for token in tokens:\n            token = token.lower()\n            word = tuple(token)\n            word = tuple(list(word[:-1]) + [word[-1] + \"</w>\"])\n            pairs = get_pairs(word)\n\n            if not pairs:\n                words.append(token)\n                continue\n\n            while True:\n                bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float(\"inf\")))\n                if bigram not in self.bpe_ranks:\n                    break\n                first, second = bigram\n                new_word = []\n                i = 0\n\n                while i < len(word):\n                    try:\n                        j = word.index(first, i)\n                        new_word.extend(word[i:j])\n                        i = j\n                    except ValueError:\n                        new_word.extend(word[i:])\n                        break\n\n                    if word[i] == first and i < len(word) - 1 and word[i + 1] == second:\n                        new_word.append(first + second)\n                        i += 2\n                    else:\n                        new_word.append(word[i])\n                        i += 1\n                new_word = tuple(new_word)\n                word = new_word\n                if len(word) == 1:\n                    break\n                else:\n                    pairs = get_pairs(word)\n            word = \"@@ \".join(word)\n            word = word[:-4]\n\n            self.cache[token] = word\n            words.append(word)\n        return \" \".join(words)\n\n    def _tokenize(self, text: str) -> List[str]:\n        \"\"\" Split a string into tokens using BPE.\"\"\"\n        split_tokens = []\n\n        words = re.findall(r\"\\S+\\n?\", text)\n\n        for token in words:\n            split_tokens.extend([t for t in self.bpe(token).split(\" \")])\n        return split_tokens\n\n    def _convert_token_to_id(self, token: str) -> int:\n        \"\"\" Converts a token to an id using the vocab. \"\"\"\n        token = token.lower()\n        return self.encoder.get(token, self.encoder.get(self.unk_token))\n\n    def _convert_id_to_token(self, index: int) -> str:\n        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n        return self.decoder.get(index, self.unk_token)\n\n    def convert_tokens_to_string(self, tokens: List[str]) -> str:\n        \"\"\" Converts a sequence of tokens  in a single string. \"\"\"\n        out_string = \" \".join(tokens).replace(\"@@ \", \"\").strip()\n        return out_string\n\n    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        vocab_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"vocab_file\"]\n        )\n        merge_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"merges_file\"]\n        )\n\n        with open(vocab_file, \"w\", encoding=\"utf-8\") as f:\n            f.write(json.dumps(self.encoder, ensure_ascii=False))\n\n        index = 0\n        with open(merge_file, \"w\", encoding=\"utf-8\") as writer:\n            writer.write(\"#version: 0.2\\n\")\n            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):\n                if index != token_index:\n                    logger.warning(\n                        \"Saving vocabulary to {}: BPE merge indices are not consecutive.\"\n                        \" Please check that the tokenizer is not corrupted!\".format(merge_file)\n                    )\n                    index = token_index\n                writer.write(\" \".join(bpe_tokens) + \"\\n\")\n                index += 1\n\n        return vocab_file, merge_file\n", "path": "src/transformers/tokenization_blenderbot.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n# coding=utf-8\n# Copyright (c) Facebook, Inc. and its affiliates.\n#\n# This source code is licensed under the MIT license found in the;\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# LICENSE file in the root directory of this source tree.\n\"\"\"\"BlenderbotTokenizer and BlenderbotSmallTokenizer\"\"\"\nimport json\nimport os\nfrom typing import Dict, List, Optional, Tuple\n\nimport regex as re\n\nfrom .tokenization_roberta import RobertaTokenizer\nfrom .tokenization_utils import PreTrainedTokenizer\nfrom .utils import logging\n\n\nlogger = logging.get_logger(__name__)\n\n\nVOCAB_FILES_NAMES = {\n    \"vocab_file\": \"vocab.json\",\n    \"merges_file\": \"merges.txt\",\n    # \"tokenizer_config_file\": \"tokenizer_config.json\",\n}\nCKPT_3B = \"facebook/blenderbot-3B\"\n\n\nclass BlenderbotTokenizer(RobertaTokenizer):\n    r\"\"\"\n    Construct a Blenderbot tokenizer.\n\n    :class:`~transformers.Blenderbot` is nearly identical to :class:`~transformers.RobertaTokenizer` and runs\n    end-to-end tokenization: punctuation splitting and wordpiece. The only difference is that it doesnt add BOS\n    token to the beginning of sequences.\n\n    Refer to superclass :class:`~transformers.RobertaTokenizer` for usage examples and documentation concerning\n    parameters.\n    \"\"\"\n    vocab_files_names = {\n        \"vocab_file\": \"vocab.json\",\n        \"merges_file\": \"merges.txt\",\n        \"tokenizer_config_file\": \"tokenizer_config.json\",\n    }\n    pretrained_vocab_files_map = {\n        \"vocab_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json\"},\n        \"merges_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt\"},\n        \"tokenizer_config_file\": {CKPT_3B: \"https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json\"},\n    }\n    max_model_input_sizes = {\"facebook/blenderbot-3B\": 128}\n\n    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: List[int] = None):\n        \"\"\"\n        Build model inputs from a sequence or a pair of sequence for sequence classification tasks\n        by concatenating and adding special tokens.\n        A Blenderbot sequence has the following format:\n\n        - single sequence: `` X </s>``\n\n        Args:\n            token_ids_0 (:obj:`List[int]`):\n                List of IDs to which the special tokens will be added\n            token_ids_1 (:obj:`List[int]`, `optional`):\n                Will be ignored\n\n        Returns:\n            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.\n        \"\"\"\n        return token_ids_0 + [self.eos_token_id]\n\n\ndef get_pairs(word):\n    \"\"\"Return set of symbol pairs in a word.\n\n    Word is represented as tuple of symbols (symbols being variable-length strings).\n    \"\"\"\n    pairs = set()\n    prev_char = word[0]\n    for char in word[1:]:\n        pairs.add((prev_char, char))\n        prev_char = char\n\n    pairs = set(pairs)\n    return pairs\n\n\nclass BlenderbotSmallTokenizer(PreTrainedTokenizer):\n    \"\"\"\n    Constructs a Blenderbot-90M tokenizer based on BPE (Byte-Pair-Encoding)\n\n    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods. Users\n    should refer to the superclass for more information regarding methods.\n\n    Args:\n        vocab_file (:obj:`str`):\n            File containing the vocabulary.\n        merges_file (:obj:`str`):\n            Path to the merges file.\n        bos_token (:obj:`str`, `optional`, defaults to :obj:`\"__start__\"`):\n            The beginning of sentence token.\n        eos_token (:obj:`str`, `optional`, defaults to :obj:`\"__end__\"`):\n            The end of sentence token.\n        unk_token (:obj:`str`, `optional`, defaults to :obj:`\"__unk__\"`):\n            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead.\n        pad_token (:obj:`str`, `optional`, defaults to :obj:`\"__pad__\"`):\n            The token used for padding, for example when batching sequences of different lengths.\n        **kwargs\n            Additional keyword arguments passed along to  :class:`~transformers.PreTrainedTokenizer`\n    \"\"\"\n\n    vocab_files_names = {\"vocab_file\": \"vocab.json\", \"merges_file\": \"merges.txt\"}\n    pretrained_vocab_files_map = {\n        \"vocab_file\": {\"facebook/blenderbot-90M\": \"https://cdn.huggingface.co/facebook/blenderbot-90M/vocab.json\"},\n        \"merges_file\": {\"facebook/blenderbot-90M\": \"https://cdn.huggingface.co/facebook/blenderbot-90M/merges.txt\"},\n    }\n    max_model_input_sizes = {\"facebook/blenderbot-90M\": 512}\n\n    def __init__(\n        self,\n        vocab_file,\n        merges_file,\n        bos_token=\"__start__\",\n        eos_token=\"__end__\",\n        unk_token=\"__unk__\",\n        pad_token=\"__null\",\n        **kwargs\n    ):\n        super().__init__(unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, pad_token=pad_token, **kwargs)\n\n        with open(vocab_file, encoding=\"utf-8\") as vocab_handle:\n            self.encoder = json.load(vocab_handle)\n        self.decoder = {v: k for k, v in self.encoder.items()}\n        with open(merges_file, encoding=\"utf-8\") as merges_handle:\n            merges = merges_handle.read().split(\"\\n\")[1:-1]\n        merges = [tuple(merge.split()) for merge in merges]\n        self.bpe_ranks = dict(zip(merges, range(len(merges))))\n        self.cache = {}\n\n    @property\n    def vocab_size(self) -> int:\n        return len(self.encoder)\n\n    def get_vocab(self) -> Dict:\n        return dict(self.encoder, **self.added_tokens_encoder)\n\n    def bpe(self, token: str) -> str:\n        if token in self.cache:\n            return self.cache[token]\n        token = re.sub(\"([.,!?()])\", r\" \\1\", token)\n        token = re.sub(\"(')\", r\" \\1 \", token)\n        token = re.sub(\"\\s{2,}\", \" \", token)\n        if \"\\n\" in token:\n            token = token.replace(\"\\n\", \" __newln__\")\n\n        tokens = token.split(\" \")\n        words = []\n        for token in tokens:\n            if not len(token):\n                continue\n\n            token = token.lower()\n            word = tuple(token)\n            word = tuple(list(word[:-1]) + [word[-1] + \"</w>\"])\n            pairs = get_pairs(word)\n\n            if not pairs:\n                words.append(token)\n                continue\n\n            while True:\n                bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float(\"inf\")))\n                if bigram not in self.bpe_ranks:\n                    break\n                first, second = bigram\n                new_word = []\n                i = 0\n\n                while i < len(word):\n                    try:\n                        j = word.index(first, i)\n                        new_word.extend(word[i:j])\n                        i = j\n                    except ValueError:\n                        new_word.extend(word[i:])\n                        break\n\n                    if word[i] == first and i < len(word) - 1 and word[i + 1] == second:\n                        new_word.append(first + second)\n                        i += 2\n                    else:\n                        new_word.append(word[i])\n                        i += 1\n                new_word = tuple(new_word)\n                word = new_word\n                if len(word) == 1:\n                    break\n                else:\n                    pairs = get_pairs(word)\n            word = \"@@ \".join(word)\n            word = word[:-4]\n\n            self.cache[token] = word\n            words.append(word)\n        return \" \".join(words)\n\n    def _tokenize(self, text: str) -> List[str]:\n        \"\"\" Split a string into tokens using BPE.\"\"\"\n        split_tokens = []\n\n        words = re.findall(r\"\\S+\\n?\", text)\n\n        for token in words:\n            split_tokens.extend([t for t in self.bpe(token).split(\" \")])\n        return split_tokens\n\n    def _convert_token_to_id(self, token: str) -> int:\n        \"\"\" Converts a token to an id using the vocab. \"\"\"\n        token = token.lower()\n        return self.encoder.get(token, self.encoder.get(self.unk_token))\n\n    def _convert_id_to_token(self, index: int) -> str:\n        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n        return self.decoder.get(index, self.unk_token)\n\n    def convert_tokens_to_string(self, tokens: List[str]) -> str:\n        \"\"\" Converts a sequence of tokens  in a single string. \"\"\"\n        out_string = \" \".join(tokens).replace(\"@@ \", \"\").strip()\n        return out_string\n\n    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:\n        if not os.path.isdir(save_directory):\n            logger.error(\"Vocabulary path ({}) should be a directory\".format(save_directory))\n            return\n        vocab_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"vocab_file\"]\n        )\n        merge_file = os.path.join(\n            save_directory, (filename_prefix + \"-\" if filename_prefix else \"\") + VOCAB_FILES_NAMES[\"merges_file\"]\n        )\n\n        with open(vocab_file, \"w\", encoding=\"utf-8\") as f:\n            f.write(json.dumps(self.encoder, ensure_ascii=False))\n\n        index = 0\n        with open(merge_file, \"w\", encoding=\"utf-8\") as writer:\n            writer.write(\"#version: 0.2\\n\")\n            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):\n                if index != token_index:\n                    logger.warning(\n                        \"Saving vocabulary to {}: BPE merge indices are not consecutive.\"\n                        \" Please check that the tokenizer is not corrupted!\".format(merge_file)\n                    )\n                    index = token_index\n                writer.write(\" \".join(bpe_tokens) + \"\\n\")\n                index += 1\n\n        return vocab_file, merge_file\n", "path": "src/transformers/tokenization_blenderbot.py"}]}