We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
minloglevel issue with SentencePiece
Hi @aheba,

It's not the first time that I face this problem, and it's a big one on Cedar (could be the case on other clusters). SentencePiece 0.1.92 cannot be installed (as it is yanked according to the official GitHub, the version has some big regression bugs). Problem: 0.1.91 doesn't accept minloglevel as an argument raising an error 

**OSError: Not found: unknown field name "minloglevel" in TrainerSpec.**
I suppose that we have to remove this option so it always work, but the logs are terrible ... any ideas ?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `speechbrain/tokenizers/SentencePiece.py`
Content:
```
1 """Library for Byte-pair-encoding (BPE) tokenization.
2 
3 Authors
4  * Abdelwahab Heba 2020
5 """
6 
7 import os.path
8 import torch
9 import logging
10 import csv
11 import sentencepiece as spm
12 from speechbrain.data_io.data_io import merge_char
13 
14 logger = logging.getLogger(__name__)
15 
16 
17 class SentencePiece:
18     """
19     BPE class call the SentencePiece unsupervised text tokenizer from Google.
20     Ref: https://github.com/google/sentencepiece
21 
22     SetencePiece lib is an unsupervised text tokenizer and detokenizer.
23     It implements subword units like Byte-pair-encoding (BPE),
24     Unigram language model and char/word tokenizer.
25 
26     Arguments
27     ---------
28     model_dir: str
29         The directory where the model is saved.
30     vocab_size: int, None, optional
31         Vocab size for the choosen tokenizer type (BPE, Unigram).
32         The vocab_size is optional for char, and mandatory for BPE & unigram
33         tokenization.
34     csv_train: str
35         Path of the csv file which is used for learn of create the tokenizer.
36     csv_read: str
37         The data entry which contain the word sequence in the csv file.
38     model_type: str
39         (bpe, char, unigram).
40         If "bpe", train unsupervised tokenization of piece of words. see:
41         https://www.aclweb.org/anthology/P16-1162/
42         If "word" take the vocabulary from the input text.
43         If "unigram" do piece of word tokenization using unigram language
44         model, see: https://arxiv.org/abs/1804.10959
45     char_format_input : bool
46         Default : False
47         Whether the csv_read entry contains characters format input.
48         (ex. a p p l e _ i s _ g o o d)
49     character_coverage: int
50         Default: 1.0, Amount of characters covered by the model, good defaults
51         are: 0.9995 for languages with rich character set like Japanse or
52         Chinese and 1.0 for other languages with small character set.
53     max_sentencepiece_length: int
54         Deault: 10,
55         Maximum number of characters for the tokens.
56     bos_id: int
57         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.
58     eos_id: int
59         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.
60     minloglevel: int
61         Default: 1, if 1 then the log(INFO) for BPE training is skipped
62         if 0, the log is printed.
63 
64     Example
65     -------
66     >>> import torch
67     >>> dict_int2lab = {1: "HELLO", 2: "MORNING"}
68     >>> model_dir = "tests/unittests/tokenizer_data/"
69     >>> csv_train = "tests/unittests/tokenizer_data/dev-clean.csv"
70     >>> csv_read = "wrd"
71     >>> model_type = "bpe"
72     >>> bpe = SentencePiece(model_dir,2000, csv_train, csv_read, model_type)
73     >>> batch_seq = torch.Tensor([[1, 2, 2, 1],[1, 2, 1, 0]])
74     >>> batch_lens = torch.Tensor([1.0, 0.75])
75     >>> encoded_seq_ids, encoded_seq_pieces = bpe(
76     ...     batch_seq, batch_lens, dict_int2lab, task="encode"
77     ... )
78     """
79 
80     def __init__(
81         self,
82         model_dir,
83         vocab_size,
84         csv_train=None,
85         csv_read=None,
86         model_type="unigram",
87         char_format_input=False,
88         character_coverage=1.0,
89         max_sentencepiece_length=10,
90         bos_id=-1,
91         eos_id=-1,
92         pad_id=-1,
93         unk_id=0,
94         minloglevel=1,
95     ):
96         if model_type not in ["unigram", "bpe", "char"]:
97             raise ValueError("model_type must be one of : [unigram, bpe, char]")
98         if not os.path.isdir(model_dir):
99             os.makedirs(model_dir)
100         if not isinstance(vocab_size, int):
101             raise ValueError("vocab_size must be integer.")
102 
103         self.csv_train = csv_train
104         self.csv_read = csv_read
105         self.text_file = os.path.join(
106             os.path.dirname(csv_train),
107             os.path.splitext(os.path.basename(csv_train))[0] + ".txt",
108         )
109         self.prefix_model_file = os.path.join(
110             model_dir, str(vocab_size) + "_" + model_type
111         )
112         self.vocab_size = str(vocab_size)
113         self.model_type = model_type
114         self.char_format_input = char_format_input
115         self.character_coverage = str(character_coverage)
116         self.max_sentencepiece_length = str(max_sentencepiece_length)
117         self.bos_id = str(bos_id)
118         self.eos_id = str(eos_id)
119         self.pad_id = str(pad_id)
120         self.unk_id = str(unk_id)
121         self.minloglevel = str(minloglevel)
122 
123         if not os.path.isfile(self.prefix_model_file + ".model"):
124             logger.info("Train tokenizer with type:" + self.model_type)
125             if not os.path.isfile(self.text_file):
126                 self._csv2text()
127             self._train_BPE()
128         else:
129             logger.info("Tokenizer is already trained.")
130         logger.info("==== Loading Tokenizer ===")
131         logger.info("Tokenizer path: " + self.prefix_model_file + ".model")
132         logger.info("Tokenizer vocab_size: " + str(self.vocab_size))
133         logger.info("Tokenizer type: " + self.model_type)
134         self.sp = spm.SentencePieceProcessor()
135         self.sp.load(self.prefix_model_file + ".model")
136 
137     def _csv2text(self):
138         """
139         Read CSV file and convert specific data entries into text file.
140         """
141         if not os.path.isfile(os.path.abspath(self.csv_train)):
142             raise ValueError(
143                 self.csv_train
144                 + " is not a file. please provide csv file for training."
145             )
146         logger.info(
147             "Extract " + self.csv_read + " sequences from:" + self.csv_train
148         )
149         csv_file = open(self.csv_train, "r")
150         reader = csv.reader(csv_file)
151         headers = next(reader, None)
152         if self.csv_read not in headers:
153             raise ValueError(self.csv_read + "must exist in:" + self.csv_train)
154         index_label = headers.index(self.csv_read)
155         text_file = open(self.text_file, "w+")
156         for row in reader:
157             sent = row[index_label]
158             if self.char_format_input:
159                 (sent,) = merge_char([sent.split()])
160                 sent = " ".join(sent)
161             text_file.write(sent + "\n")
162         text_file.close()
163         csv_file.close()
164         logger.info("Text file created at: " + self.text_file)
165 
166     def _train_BPE(self):
167         """
168         Train tokenizer with unsupervised techniques (BPE, Unigram) using
169         SentencePiece Library. If you use "char" mode, the SentencePiece
170         creates a char dict so the vocab_size attribute is not needed.
171         """
172         query = (
173             "--input="
174             + self.text_file
175             + " --model_prefix="
176             + self.prefix_model_file
177             + " --model_type="
178             + self.model_type
179             + " --bos_id="
180             + self.bos_id
181             + " --eos_id="
182             + self.eos_id
183             + " --pad_id="
184             + self.pad_id
185             + " --unk_id="
186             + self.unk_id
187             + " --max_sentencepiece_length="
188             + self.max_sentencepiece_length
189             + " --character_coverage="
190             + self.character_coverage
191             + " --minloglevel="
192             + self.minloglevel
193         )
194         if self.model_type not in ["char"]:
195             # include vocab_size
196             query += " --vocab_size=" + str(self.vocab_size)
197         # Train tokenizer
198         spm.SentencePieceTrainer.train(query)
199 
200     def __call__(
201         self, batch, batch_lens=None, ind2lab=None, task="encode",
202     ):
203         """
204         This __call__ function implements the tokenizer encoder and decoder
205         (restoring the string of word) for BPE, Regularized BPE (with unigram),
206         and char (speechbrain/nnet/RNN.py).
207 
208         Arguments
209         ----------
210         batch : tensor.IntTensor or list
211             list if ( batch_lens = None and task = "decode_from_list")
212             Contains the original labels. Shape: [batch_size, max_length]
213         batch_lens : tensor.LongTensor
214             Default: None,
215             Cotaining the relative length of each label sequences. Must be 1D
216             tensor of shape: [batch_size].
217         ind2lab : dict
218             Dictionnary which map the index from label sequences
219             (batch tensor) to string label.
220         task: str
221             ("encode", "decode", "decode_from_list)
222             "encode": convert the batch tensor into sequence of tokens.
223                 the output contain a list of (tokens_seq, tokens_lens)
224             "decode": convert a tensor of tokens to a list of word sequences.
225             "decode_from_list": convert a list of token sequences to a list
226                 of word sequences.
227         """
228         if task == "encode" and ind2lab is None:
229             raise ValueError("Tokenizer encoder must have the ind2lab function")
230 
231         if task == "encode":
232             # Convert list of words/chars to bpe ids
233             bpe = []
234             max_bpe_len = 0
235             batch_lens = (batch_lens * batch.shape[1]).int()
236             for i, utt_seq in enumerate(batch):
237                 tokens = [
238                     ind2lab[int(index)] for index in utt_seq[: batch_lens[i]]
239                 ]
240                 if self.char_format_input:
241                     (words_list,) = merge_char([tokens])
242                     sent = " ".join(words_list)
243                 else:
244                     sent = " ".join(tokens)
245                 bpe_encode = self.sp.encode_as_ids(sent)
246                 bpe.append(bpe_encode)
247                 # save the longest bpe sequence
248                 # it help to compute the relative length of each utterance
249                 if len(bpe_encode) > max_bpe_len:
250                     max_bpe_len = len(bpe_encode)
251             # Create bpe tensor
252             bpe_tensor = torch.zeros(
253                 (batch.shape[0], max_bpe_len), device=batch.device
254             )
255             bpe_lens = torch.zeros((batch.shape[0]), device=batch.device)
256             for i, bpe_utt in enumerate(bpe):
257                 bpe_tensor[i, : len(bpe_utt)] = torch.Tensor(bpe_utt)
258                 bpe_lens[i] = len(bpe_utt) / max_bpe_len
259             return bpe_tensor, bpe_lens
260         elif task == "decode_from_list":
261             # From list of hyps (not padded outputs)
262             # do decoding
263             return [self.sp.decode_ids(utt_seq).split(" ") for utt_seq in batch]
264         elif task == "decode":
265             # From a batch tensor and a length tensor
266             # find the absolute batch lengths and do decoding
267             batch_lens = (batch_lens * batch.shape[1]).int()
268             return [
269                 self.sp.decode_ids(
270                     utt_seq[: batch_lens[i]].int().tolist()
271                 ).split(" ")
272                 for i, utt_seq in enumerate(batch)
273             ]
274
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/speechbrain/tokenizers/SentencePiece.py b/speechbrain/tokenizers/SentencePiece.py
--- a/speechbrain/tokenizers/SentencePiece.py
+++ b/speechbrain/tokenizers/SentencePiece.py
@@ -57,9 +57,6 @@
         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.
     eos_id: int
         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.
-    minloglevel: int
-        Default: 1, if 1 then the log(INFO) for BPE training is skipped
-        if 0, the log is printed.
 
     Example
     -------
@@ -91,7 +88,6 @@
         eos_id=-1,
         pad_id=-1,
         unk_id=0,
-        minloglevel=1,
     ):
         if model_type not in ["unigram", "bpe", "char"]:
             raise ValueError("model_type must be one of : [unigram, bpe, char]")
@@ -118,7 +114,6 @@
         self.eos_id = str(eos_id)
         self.pad_id = str(pad_id)
         self.unk_id = str(unk_id)
-        self.minloglevel = str(minloglevel)
 
         if not os.path.isfile(self.prefix_model_file + ".model"):
             logger.info("Train tokenizer with type:" + self.model_type)
@@ -188,8 +183,6 @@
             + self.max_sentencepiece_length
             + " --character_coverage="
             + self.character_coverage
-            + " --minloglevel="
-            + self.minloglevel
         )
         if self.model_type not in ["char"]:
             # include vocab_size

{"golden_diff": "diff --git a/speechbrain/tokenizers/SentencePiece.py b/speechbrain/tokenizers/SentencePiece.py\n--- a/speechbrain/tokenizers/SentencePiece.py\n+++ b/speechbrain/tokenizers/SentencePiece.py\n@@ -57,9 +57,6 @@\n         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n     eos_id: int\n         Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n-    minloglevel: int\n-        Default: 1, if 1 then the log(INFO) for BPE training is skipped\n-        if 0, the log is printed.\n \n     Example\n     -------\n@@ -91,7 +88,6 @@\n         eos_id=-1,\n         pad_id=-1,\n         unk_id=0,\n-        minloglevel=1,\n     ):\n         if model_type not in [\"unigram\", \"bpe\", \"char\"]:\n             raise ValueError(\"model_type must be one of : [unigram, bpe, char]\")\n@@ -118,7 +114,6 @@\n         self.eos_id = str(eos_id)\n         self.pad_id = str(pad_id)\n         self.unk_id = str(unk_id)\n-        self.minloglevel = str(minloglevel)\n \n         if not os.path.isfile(self.prefix_model_file + \".model\"):\n             logger.info(\"Train tokenizer with type:\" + self.model_type)\n@@ -188,8 +183,6 @@\n             + self.max_sentencepiece_length\n             + \" --character_coverage=\"\n             + self.character_coverage\n-            + \" --minloglevel=\"\n-            + self.minloglevel\n         )\n         if self.model_type not in [\"char\"]:\n             # include vocab_size\n", "issue": "minloglevel issue with SentencePiece\nHi @aheba,\r\n\r\nIt's not the first time that I face this problem, and it's a big one on Cedar (could be the case on other clusters). SentencePiece 0.1.92 cannot be installed (as it is yanked according to the official GitHub, the version has some big regression bugs). Problem: 0.1.91 doesn't accept minloglevel as an argument raising an error \r\n\r\n**OSError: Not found: unknown field name \"minloglevel\" in TrainerSpec.**\r\nI suppose that we have to remove this option so it always work, but the logs are terrible ... any ideas ?\n", "before_files": [{"content": "\"\"\"Library for Byte-pair-encoding (BPE) tokenization.\n\nAuthors\n * Abdelwahab Heba 2020\n\"\"\"\n\nimport os.path\nimport torch\nimport logging\nimport csv\nimport sentencepiece as spm\nfrom speechbrain.data_io.data_io import merge_char\n\nlogger = logging.getLogger(__name__)\n\n\nclass SentencePiece:\n    \"\"\"\n    BPE class call the SentencePiece unsupervised text tokenizer from Google.\n    Ref: https://github.com/google/sentencepiece\n\n    SetencePiece lib is an unsupervised text tokenizer and detokenizer.\n    It implements subword units like Byte-pair-encoding (BPE),\n    Unigram language model and char/word tokenizer.\n\n    Arguments\n    ---------\n    model_dir: str\n        The directory where the model is saved.\n    vocab_size: int, None, optional\n        Vocab size for the choosen tokenizer type (BPE, Unigram).\n        The vocab_size is optional for char, and mandatory for BPE & unigram\n        tokenization.\n    csv_train: str\n        Path of the csv file which is used for learn of create the tokenizer.\n    csv_read: str\n        The data entry which contain the word sequence in the csv file.\n    model_type: str\n        (bpe, char, unigram).\n        If \"bpe\", train unsupervised tokenization of piece of words. see:\n        https://www.aclweb.org/anthology/P16-1162/\n        If \"word\" take the vocabulary from the input text.\n        If \"unigram\" do piece of word tokenization using unigram language\n        model, see: https://arxiv.org/abs/1804.10959\n    char_format_input : bool\n        Default : False\n        Whether the csv_read entry contains characters format input.\n        (ex. a p p l e _ i s _ g o o d)\n    character_coverage: int\n        Default: 1.0, Amount of characters covered by the model, good defaults\n        are: 0.9995 for languages with rich character set like Japanse or\n        Chinese and 1.0 for other languages with small character set.\n    max_sentencepiece_length: int\n        Deault: 10,\n        Maximum number of characters for the tokens.\n    bos_id: int\n        Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n    eos_id: int\n        Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n    minloglevel: int\n        Default: 1, if 1 then the log(INFO) for BPE training is skipped\n        if 0, the log is printed.\n\n    Example\n    -------\n    >>> import torch\n    >>> dict_int2lab = {1: \"HELLO\", 2: \"MORNING\"}\n    >>> model_dir = \"tests/unittests/tokenizer_data/\"\n    >>> csv_train = \"tests/unittests/tokenizer_data/dev-clean.csv\"\n    >>> csv_read = \"wrd\"\n    >>> model_type = \"bpe\"\n    >>> bpe = SentencePiece(model_dir,2000, csv_train, csv_read, model_type)\n    >>> batch_seq = torch.Tensor([[1, 2, 2, 1],[1, 2, 1, 0]])\n    >>> batch_lens = torch.Tensor([1.0, 0.75])\n    >>> encoded_seq_ids, encoded_seq_pieces = bpe(\n    ...     batch_seq, batch_lens, dict_int2lab, task=\"encode\"\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        model_dir,\n        vocab_size,\n        csv_train=None,\n        csv_read=None,\n        model_type=\"unigram\",\n        char_format_input=False,\n        character_coverage=1.0,\n        max_sentencepiece_length=10,\n        bos_id=-1,\n        eos_id=-1,\n        pad_id=-1,\n        unk_id=0,\n        minloglevel=1,\n    ):\n        if model_type not in [\"unigram\", \"bpe\", \"char\"]:\n            raise ValueError(\"model_type must be one of : [unigram, bpe, char]\")\n        if not os.path.isdir(model_dir):\n            os.makedirs(model_dir)\n        if not isinstance(vocab_size, int):\n            raise ValueError(\"vocab_size must be integer.\")\n\n        self.csv_train = csv_train\n        self.csv_read = csv_read\n        self.text_file = os.path.join(\n            os.path.dirname(csv_train),\n            os.path.splitext(os.path.basename(csv_train))[0] + \".txt\",\n        )\n        self.prefix_model_file = os.path.join(\n            model_dir, str(vocab_size) + \"_\" + model_type\n        )\n        self.vocab_size = str(vocab_size)\n        self.model_type = model_type\n        self.char_format_input = char_format_input\n        self.character_coverage = str(character_coverage)\n        self.max_sentencepiece_length = str(max_sentencepiece_length)\n        self.bos_id = str(bos_id)\n        self.eos_id = str(eos_id)\n        self.pad_id = str(pad_id)\n        self.unk_id = str(unk_id)\n        self.minloglevel = str(minloglevel)\n\n        if not os.path.isfile(self.prefix_model_file + \".model\"):\n            logger.info(\"Train tokenizer with type:\" + self.model_type)\n            if not os.path.isfile(self.text_file):\n                self._csv2text()\n            self._train_BPE()\n        else:\n            logger.info(\"Tokenizer is already trained.\")\n        logger.info(\"==== Loading Tokenizer ===\")\n        logger.info(\"Tokenizer path: \" + self.prefix_model_file + \".model\")\n        logger.info(\"Tokenizer vocab_size: \" + str(self.vocab_size))\n        logger.info(\"Tokenizer type: \" + self.model_type)\n        self.sp = spm.SentencePieceProcessor()\n        self.sp.load(self.prefix_model_file + \".model\")\n\n    def _csv2text(self):\n        \"\"\"\n        Read CSV file and convert specific data entries into text file.\n        \"\"\"\n        if not os.path.isfile(os.path.abspath(self.csv_train)):\n            raise ValueError(\n                self.csv_train\n                + \" is not a file. please provide csv file for training.\"\n            )\n        logger.info(\n            \"Extract \" + self.csv_read + \" sequences from:\" + self.csv_train\n        )\n        csv_file = open(self.csv_train, \"r\")\n        reader = csv.reader(csv_file)\n        headers = next(reader, None)\n        if self.csv_read not in headers:\n            raise ValueError(self.csv_read + \"must exist in:\" + self.csv_train)\n        index_label = headers.index(self.csv_read)\n        text_file = open(self.text_file, \"w+\")\n        for row in reader:\n            sent = row[index_label]\n            if self.char_format_input:\n                (sent,) = merge_char([sent.split()])\n                sent = \" \".join(sent)\n            text_file.write(sent + \"\\n\")\n        text_file.close()\n        csv_file.close()\n        logger.info(\"Text file created at: \" + self.text_file)\n\n    def _train_BPE(self):\n        \"\"\"\n        Train tokenizer with unsupervised techniques (BPE, Unigram) using\n        SentencePiece Library. If you use \"char\" mode, the SentencePiece\n        creates a char dict so the vocab_size attribute is not needed.\n        \"\"\"\n        query = (\n            \"--input=\"\n            + self.text_file\n            + \" --model_prefix=\"\n            + self.prefix_model_file\n            + \" --model_type=\"\n            + self.model_type\n            + \" --bos_id=\"\n            + self.bos_id\n            + \" --eos_id=\"\n            + self.eos_id\n            + \" --pad_id=\"\n            + self.pad_id\n            + \" --unk_id=\"\n            + self.unk_id\n            + \" --max_sentencepiece_length=\"\n            + self.max_sentencepiece_length\n            + \" --character_coverage=\"\n            + self.character_coverage\n            + \" --minloglevel=\"\n            + self.minloglevel\n        )\n        if self.model_type not in [\"char\"]:\n            # include vocab_size\n            query += \" --vocab_size=\" + str(self.vocab_size)\n        # Train tokenizer\n        spm.SentencePieceTrainer.train(query)\n\n    def __call__(\n        self, batch, batch_lens=None, ind2lab=None, task=\"encode\",\n    ):\n        \"\"\"\n        This __call__ function implements the tokenizer encoder and decoder\n        (restoring the string of word) for BPE, Regularized BPE (with unigram),\n        and char (speechbrain/nnet/RNN.py).\n\n        Arguments\n        ----------\n        batch : tensor.IntTensor or list\n            list if ( batch_lens = None and task = \"decode_from_list\")\n            Contains the original labels. Shape: [batch_size, max_length]\n        batch_lens : tensor.LongTensor\n            Default: None,\n            Cotaining the relative length of each label sequences. Must be 1D\n            tensor of shape: [batch_size].\n        ind2lab : dict\n            Dictionnary which map the index from label sequences\n            (batch tensor) to string label.\n        task: str\n            (\"encode\", \"decode\", \"decode_from_list)\n            \"encode\": convert the batch tensor into sequence of tokens.\n                the output contain a list of (tokens_seq, tokens_lens)\n            \"decode\": convert a tensor of tokens to a list of word sequences.\n            \"decode_from_list\": convert a list of token sequences to a list\n                of word sequences.\n        \"\"\"\n        if task == \"encode\" and ind2lab is None:\n            raise ValueError(\"Tokenizer encoder must have the ind2lab function\")\n\n        if task == \"encode\":\n            # Convert list of words/chars to bpe ids\n            bpe = []\n            max_bpe_len = 0\n            batch_lens = (batch_lens * batch.shape[1]).int()\n            for i, utt_seq in enumerate(batch):\n                tokens = [\n                    ind2lab[int(index)] for index in utt_seq[: batch_lens[i]]\n                ]\n                if self.char_format_input:\n                    (words_list,) = merge_char([tokens])\n                    sent = \" \".join(words_list)\n                else:\n                    sent = \" \".join(tokens)\n                bpe_encode = self.sp.encode_as_ids(sent)\n                bpe.append(bpe_encode)\n                # save the longest bpe sequence\n                # it help to compute the relative length of each utterance\n                if len(bpe_encode) > max_bpe_len:\n                    max_bpe_len = len(bpe_encode)\n            # Create bpe tensor\n            bpe_tensor = torch.zeros(\n                (batch.shape[0], max_bpe_len), device=batch.device\n            )\n            bpe_lens = torch.zeros((batch.shape[0]), device=batch.device)\n            for i, bpe_utt in enumerate(bpe):\n                bpe_tensor[i, : len(bpe_utt)] = torch.Tensor(bpe_utt)\n                bpe_lens[i] = len(bpe_utt) / max_bpe_len\n            return bpe_tensor, bpe_lens\n        elif task == \"decode_from_list\":\n            # From list of hyps (not padded outputs)\n            # do decoding\n            return [self.sp.decode_ids(utt_seq).split(\" \") for utt_seq in batch]\n        elif task == \"decode\":\n            # From a batch tensor and a length tensor\n            # find the absolute batch lengths and do decoding\n            batch_lens = (batch_lens * batch.shape[1]).int()\n            return [\n                self.sp.decode_ids(\n                    utt_seq[: batch_lens[i]].int().tolist()\n                ).split(\" \")\n                for i, utt_seq in enumerate(batch)\n            ]\n", "path": "speechbrain/tokenizers/SentencePiece.py"}], "after_files": [{"content": "\"\"\"Library for Byte-pair-encoding (BPE) tokenization.\n\nAuthors\n * Abdelwahab Heba 2020\n\"\"\"\n\nimport os.path\nimport torch\nimport logging\nimport csv\nimport sentencepiece as spm\nfrom speechbrain.data_io.data_io import merge_char\n\nlogger = logging.getLogger(__name__)\n\n\nclass SentencePiece:\n    \"\"\"\n    BPE class call the SentencePiece unsupervised text tokenizer from Google.\n    Ref: https://github.com/google/sentencepiece\n\n    SetencePiece lib is an unsupervised text tokenizer and detokenizer.\n    It implements subword units like Byte-pair-encoding (BPE),\n    Unigram language model and char/word tokenizer.\n\n    Arguments\n    ---------\n    model_dir: str\n        The directory where the model is saved.\n    vocab_size: int, None, optional\n        Vocab size for the choosen tokenizer type (BPE, Unigram).\n        The vocab_size is optional for char, and mandatory for BPE & unigram\n        tokenization.\n    csv_train: str\n        Path of the csv file which is used for learn of create the tokenizer.\n    csv_read: str\n        The data entry which contain the word sequence in the csv file.\n    model_type: str\n        (bpe, char, unigram).\n        If \"bpe\", train unsupervised tokenization of piece of words. see:\n        https://www.aclweb.org/anthology/P16-1162/\n        If \"word\" take the vocabulary from the input text.\n        If \"unigram\" do piece of word tokenization using unigram language\n        model, see: https://arxiv.org/abs/1804.10959\n    char_format_input : bool\n        Default : False\n        Whether the csv_read entry contains characters format input.\n        (ex. a p p l e _ i s _ g o o d)\n    character_coverage: int\n        Default: 1.0, Amount of characters covered by the model, good defaults\n        are: 0.9995 for languages with rich character set like Japanse or\n        Chinese and 1.0 for other languages with small character set.\n    max_sentencepiece_length: int\n        Deault: 10,\n        Maximum number of characters for the tokens.\n    bos_id: int\n        Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n    eos_id: int\n        Default: -1, if -1 the bos_id = unk_id = 0. otherwise, bos_id = int.\n\n    Example\n    -------\n    >>> import torch\n    >>> dict_int2lab = {1: \"HELLO\", 2: \"MORNING\"}\n    >>> model_dir = \"tests/unittests/tokenizer_data/\"\n    >>> csv_train = \"tests/unittests/tokenizer_data/dev-clean.csv\"\n    >>> csv_read = \"wrd\"\n    >>> model_type = \"bpe\"\n    >>> bpe = SentencePiece(model_dir,2000, csv_train, csv_read, model_type)\n    >>> batch_seq = torch.Tensor([[1, 2, 2, 1],[1, 2, 1, 0]])\n    >>> batch_lens = torch.Tensor([1.0, 0.75])\n    >>> encoded_seq_ids, encoded_seq_pieces = bpe(\n    ...     batch_seq, batch_lens, dict_int2lab, task=\"encode\"\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        model_dir,\n        vocab_size,\n        csv_train=None,\n        csv_read=None,\n        model_type=\"unigram\",\n        char_format_input=False,\n        character_coverage=1.0,\n        max_sentencepiece_length=10,\n        bos_id=-1,\n        eos_id=-1,\n        pad_id=-1,\n        unk_id=0,\n    ):\n        if model_type not in [\"unigram\", \"bpe\", \"char\"]:\n            raise ValueError(\"model_type must be one of : [unigram, bpe, char]\")\n        if not os.path.isdir(model_dir):\n            os.makedirs(model_dir)\n        if not isinstance(vocab_size, int):\n            raise ValueError(\"vocab_size must be integer.\")\n\n        self.csv_train = csv_train\n        self.csv_read = csv_read\n        self.text_file = os.path.join(\n            os.path.dirname(csv_train),\n            os.path.splitext(os.path.basename(csv_train))[0] + \".txt\",\n        )\n        self.prefix_model_file = os.path.join(\n            model_dir, str(vocab_size) + \"_\" + model_type\n        )\n        self.vocab_size = str(vocab_size)\n        self.model_type = model_type\n        self.char_format_input = char_format_input\n        self.character_coverage = str(character_coverage)\n        self.max_sentencepiece_length = str(max_sentencepiece_length)\n        self.bos_id = str(bos_id)\n        self.eos_id = str(eos_id)\n        self.pad_id = str(pad_id)\n        self.unk_id = str(unk_id)\n\n        if not os.path.isfile(self.prefix_model_file + \".model\"):\n            logger.info(\"Train tokenizer with type:\" + self.model_type)\n            if not os.path.isfile(self.text_file):\n                self._csv2text()\n            self._train_BPE()\n        else:\n            logger.info(\"Tokenizer is already trained.\")\n        logger.info(\"==== Loading Tokenizer ===\")\n        logger.info(\"Tokenizer path: \" + self.prefix_model_file + \".model\")\n        logger.info(\"Tokenizer vocab_size: \" + str(self.vocab_size))\n        logger.info(\"Tokenizer type: \" + self.model_type)\n        self.sp = spm.SentencePieceProcessor()\n        self.sp.load(self.prefix_model_file + \".model\")\n\n    def _csv2text(self):\n        \"\"\"\n        Read CSV file and convert specific data entries into text file.\n        \"\"\"\n        if not os.path.isfile(os.path.abspath(self.csv_train)):\n            raise ValueError(\n                self.csv_train\n                + \" is not a file. please provide csv file for training.\"\n            )\n        logger.info(\n            \"Extract \" + self.csv_read + \" sequences from:\" + self.csv_train\n        )\n        csv_file = open(self.csv_train, \"r\")\n        reader = csv.reader(csv_file)\n        headers = next(reader, None)\n        if self.csv_read not in headers:\n            raise ValueError(self.csv_read + \"must exist in:\" + self.csv_train)\n        index_label = headers.index(self.csv_read)\n        text_file = open(self.text_file, \"w+\")\n        for row in reader:\n            sent = row[index_label]\n            if self.char_format_input:\n                (sent,) = merge_char([sent.split()])\n                sent = \" \".join(sent)\n            text_file.write(sent + \"\\n\")\n        text_file.close()\n        csv_file.close()\n        logger.info(\"Text file created at: \" + self.text_file)\n\n    def _train_BPE(self):\n        \"\"\"\n        Train tokenizer with unsupervised techniques (BPE, Unigram) using\n        SentencePiece Library. If you use \"char\" mode, the SentencePiece\n        creates a char dict so the vocab_size attribute is not needed.\n        \"\"\"\n        query = (\n            \"--input=\"\n            + self.text_file\n            + \" --model_prefix=\"\n            + self.prefix_model_file\n            + \" --model_type=\"\n            + self.model_type\n            + \" --bos_id=\"\n            + self.bos_id\n            + \" --eos_id=\"\n            + self.eos_id\n            + \" --pad_id=\"\n            + self.pad_id\n            + \" --unk_id=\"\n            + self.unk_id\n            + \" --max_sentencepiece_length=\"\n            + self.max_sentencepiece_length\n            + \" --character_coverage=\"\n            + self.character_coverage\n        )\n        if self.model_type not in [\"char\"]:\n            # include vocab_size\n            query += \" --vocab_size=\" + str(self.vocab_size)\n        # Train tokenizer\n        spm.SentencePieceTrainer.train(query)\n\n    def __call__(\n        self, batch, batch_lens=None, ind2lab=None, task=\"encode\",\n    ):\n        \"\"\"\n        This __call__ function implements the tokenizer encoder and decoder\n        (restoring the string of word) for BPE, Regularized BPE (with unigram),\n        and char (speechbrain/nnet/RNN.py).\n\n        Arguments\n        ----------\n        batch : tensor.IntTensor or list\n            list if ( batch_lens = None and task = \"decode_from_list\")\n            Contains the original labels. Shape: [batch_size, max_length]\n        batch_lens : tensor.LongTensor\n            Default: None,\n            Cotaining the relative length of each label sequences. Must be 1D\n            tensor of shape: [batch_size].\n        ind2lab : dict\n            Dictionnary which map the index from label sequences\n            (batch tensor) to string label.\n        task: str\n            (\"encode\", \"decode\", \"decode_from_list)\n            \"encode\": convert the batch tensor into sequence of tokens.\n                the output contain a list of (tokens_seq, tokens_lens)\n            \"decode\": convert a tensor of tokens to a list of word sequences.\n            \"decode_from_list\": convert a list of token sequences to a list\n                of word sequences.\n        \"\"\"\n        if task == \"encode\" and ind2lab is None:\n            raise ValueError(\"Tokenizer encoder must have the ind2lab function\")\n\n        if task == \"encode\":\n            # Convert list of words/chars to bpe ids\n            bpe = []\n            max_bpe_len = 0\n            batch_lens = (batch_lens * batch.shape[1]).int()\n            for i, utt_seq in enumerate(batch):\n                tokens = [\n                    ind2lab[int(index)] for index in utt_seq[: batch_lens[i]]\n                ]\n                if self.char_format_input:\n                    (words_list,) = merge_char([tokens])\n                    sent = \" \".join(words_list)\n                else:\n                    sent = \" \".join(tokens)\n                bpe_encode = self.sp.encode_as_ids(sent)\n                bpe.append(bpe_encode)\n                # save the longest bpe sequence\n                # it help to compute the relative length of each utterance\n                if len(bpe_encode) > max_bpe_len:\n                    max_bpe_len = len(bpe_encode)\n            # Create bpe tensor\n            bpe_tensor = torch.zeros(\n                (batch.shape[0], max_bpe_len), device=batch.device\n            )\n            bpe_lens = torch.zeros((batch.shape[0]), device=batch.device)\n            for i, bpe_utt in enumerate(bpe):\n                bpe_tensor[i, : len(bpe_utt)] = torch.Tensor(bpe_utt)\n                bpe_lens[i] = len(bpe_utt) / max_bpe_len\n            return bpe_tensor, bpe_lens\n        elif task == \"decode_from_list\":\n            # From list of hyps (not padded outputs)\n            # do decoding\n            return [self.sp.decode_ids(utt_seq).split(\" \") for utt_seq in batch]\n        elif task == \"decode\":\n            # From a batch tensor and a length tensor\n            # find the absolute batch lengths and do decoding\n            batch_lens = (batch_lens * batch.shape[1]).int()\n            return [\n                self.sp.decode_ids(\n                    utt_seq[: batch_lens[i]].int().tolist()\n                ).split(\" \")\n                for i, utt_seq in enumerate(batch)\n            ]\n", "path": "speechbrain/tokenizers/SentencePiece.py"}]}