class_docstr: BPE class call the SentencePiece unsupervised text tokenizer from Google.
  Reference: https://github.com/google/sentencepiece
  SentencePiece lib is an unsupervised text tokenizer and detokenizer.
  It implements subword units like Byte-pair-encoding (BPE),
  Unigram language model and char/word tokenizer.
  Arguments
  ---------
  model_dir : str
      The directory where the model will be saved (or already stored).
  vocab_size : int, None, optional
      Vocab size for the chosen tokenizer type (BPE, Unigram).
      The vocab_size is optional for char, and mandatory for BPE & unigram
      tokenization.
  annotation_train : str
      Path of the annotation file which is used to learn the tokenizer. It
      can be in JSON or csv format.
  annotation_read : str
      The data entry which contains the word sequence in the annotation file.
  model_type : str
      (bpe, char, unigram).
      If "bpe", train unsupervised tokenization of piece of words. see:
      https://www.aclweb.org/anthology/P16-1162/
      If "word" take the vocabulary from the input text.
      If "unigram" do piece of word tokenization using unigram language
      model, see: https://arxiv.org/abs/1804.10959
  char_format_input : bool
      Whether the read entry contains characters format input.
      (default: False)
      (e.g., a p p l e _ i s _ g o o d)
  character_coverage : int
      Amount of characters covered by the model, good defaults
      are: 0.9995 for languages with a rich character set like Japanese or
      Chinese and 1.0 for other languages with small character set.
      (default: 1.0)
  user_defined_symbols : string
      String contained a list of symbols separated by a comma.
      User-defined symbols are handled as one piece in any context.
      (default: None)
  max_sentencepiece_length : int
      Maximum number of characters for the tokens. (default: 10)
  bos_id : int
      If -1 the bos_id = unk_id = 0. otherwise, bos_id = int. (default: -1)
  eos_id : int
      If -1 the bos_id = unk_id = 0. otherwise, bos_id = int. (default: -1)
  split_by_whitespace : bool
      If False, allow the sentencepiece to extract piece crossing multiple
      words. This feature is important for : Chinese/Japanese/Korean.
      (default: True)
  num_sequences : int
      If not none, use at most this many sequences to train the tokenizer
      (for large datasets). (default: None)
  annotation_list_to_check : list,
      List of the annotation file which is used for checking the accuracy of
      recovering words from the tokenizer.
  annotation_format : str
      The format of the annotation file. JSON or csv are the formats supported.
  text_file: str
      An alternate path to the text file (needed when multiple models are trained on
      the same data file)
  
  add_dummy_prefix : bool
      If True the tokenizer adds dummy whitespace at the beginning of text. (default: True)
  Example
  -------
  >>> import torch
  >>> dict_int2lab = {1: "HELLO", 2: "MORNING"}
  >>> model_dir = getfixture('tmpdir') / "tokenizer_data"
  >>> # Example with csv
  >>> annotation_train = "tests/samples/annotation/dev-clean.csv"
  >>> annotation_read = "wrd"
  >>> model_type = "bpe"
  >>> bpe = SentencePiece(str(model_dir), 100, annotation_train, annotation_read, model_type)
  >>> batch_seq = torch.Tensor([[1, 2, 2, 1],[1, 2, 1, 0]])
  >>> batch_lens = torch.Tensor([1.0, 0.75])
  >>> encoded_seq_ids, encoded_seq_pieces = bpe(
  ...     batch_seq, batch_lens, dict_int2lab, task="encode"
  ... )
  >>> # Example using JSON
  >>> annotation_train = str(model_dir + "/dev-clean.json")
  >>> annotation_read = "wrd"
  >>> bpe = SentencePiece(model_dir, 100, annotation_train, annotation_read, model_type, annotation_format = 'json')
  >>> encoded_seq_ids, encoded_seq_pieces = bpe(
  ...     batch_seq, batch_lens, dict_int2lab, task="encode"
  ... )
class_docstr_tok: ["BPE","class","call","the","SentencePiece","unsupervised","text","tokenizer","from","Google",".","Reference",":","https",":","\/\/github.com\/google\/sentencepiece","SentencePiece","lib","is","an","unsupervised","text","tokenizer","and","detokenizer",".","It","implements","subword","units","like","Byte-pair-encoding","(","BPE",")",",","Unigram","language","model","and","char\/word","tokenizer",".","Arguments","--","--","--","--","-","model_dir",":","str","The","directory","where","the","model","will","be","saved","(","or","already","stored",")",".","vocab_size",":","int",",","None",",","optional","Vocab","size","for","the","chosen","tokenizer","type","(","BPE",",","Unigram",")",".","The","vocab_size","is","optional","for","char",",","and","mandatory","for","BPE","&","unigram","tokenization",".","annotation_train",":","str","Path","of","the","annotation","file","which","is","used","to","learn","the","tokenizer",".","It","can","be","in","JSON","or","csv","format",".","annotation_read",":","str","The","data","entry","which","contains","the","word","sequence","in","the","annotation","file",".","model_type",":","str","(","bpe",",","char",",","unigram",")",".","If","``","bpe","''",",","train","unsupervised","tokenization","of","piece","of","words",".","see",":","https",":","\/\/www.aclweb.org\/anthology\/P16-1162\/","If","``","word","''","take","the","vocabulary","from","the","input","text",".","If","``","unigram","''","do","piece","of","word","tokenization","using","unigram","language","model",",","see",":","https",":","\/\/arxiv.org\/abs\/1804.10959","char_format_input",":","bool","Whether","the","read","entry","contains","characters","format","input",".","(","default",":","False",")","(","e.g.",",","a","p","p","l","e","_","i","s","_","g","o","o","d",")","character_coverage",":","int","Amount","of","characters","covered","by","the","model",",","good","defaults","are",":","0.9995","for","languages","with","a","rich","character","set","like","Japanese","or","Chinese","and","1.0","for","other","languages","with","small","character","set",".","(","default",":","1.0",")","user_defined_symbols",":","string","String","contained","a","list","of","symbols","separated","by","a","comma",".","User-defined","symbols","are","handled","as","one","piece","in","any","context",".","(","default",":","None",")","max_sentencepiece_length",":","int","Maximum","number","of","characters","for","the","tokens",".","(","default",":","10",")","bos_id",":","int","If","-1","the","bos_id","=","unk_id","=","0.","otherwise",",","bos_id","=","int",".","(","default",":","-1",")","eos_id",":","int","If","-1","the","bos_id","=","unk_id","=","0.","otherwise",",","bos_id","=","int",".","(","default",":","-1",")","split_by_whitespace",":","bool","If","False",",","allow","the","sentencepiece","to","extract","piece","crossing","multiple","words",".","This","feature","is","important","for",":","Chinese\/Japanese\/Korean",".","(","default",":","True",")","num_sequences",":","int","If","not","none",",","use","at","most","this","many","sequences","to","train","the","tokenizer","(","for","large","datasets",")",".","(","default",":","None",")","annotation_list_to_check",":","list",",","List","of","the","annotation","file","which","is","used","for","checking","the","accuracy","of","recovering","words","from","the","tokenizer",".","annotation_format",":","str","The","format","of","the","annotation","file",".","JSON","or","csv","are","the","formats","supported",".","text_file",":","str","An","alternate","path","to","the","text","file","(","needed","when","multiple","models","are","trained","on","the","same","data","file",")","add_dummy_prefix",":","bool","If","True","the","tokenizer","adds","dummy","whitespace","at","the","beginning","of","text",".","(","default",":","True",")","Example","--","--","--","-",">",">",">","import","torch",">",">",">","dict_int2lab","=","{","1",":","``","HELLO","''",",","2",":","``","MORNING","''","}",">",">",">","model_dir","=","getfixture","(","'tmpdir","'",")","\/","``","tokenizer_data","''",">",">",">","#","Example","with","csv",">",">",">","annotation_train","=","``","tests\/samples\/annotation\/dev-clean.csv","''",">",">",">","annotation_read","=","``","wrd","''",">",">",">","model_type","=","``","bpe","''",">",">",">","bpe","=","SentencePiece","(","str","(","model_dir",")",",","100",",","annotation_train",",","annotation_read",",","model_type",")",">",">",">","batch_seq","=","torch.Tensor","(","[","[","1",",","2",",","2",",","1","]",",","[","1",",","2",",","1",",","0","]","]",")",">",">",">","batch_lens","=","torch.Tensor","(","[","1.0",",","0.75","]",")",">",">",">","encoded_seq_ids",",","encoded_seq_pieces","=","bpe","(","...","batch_seq",",","batch_lens",",","dict_int2lab",",","task=","''","encode","''","...",")",">",">",">","#","Example","using","JSON",">",">",">","annotation_train","=","str","(","model_dir","+","``","\/dev-clean.json","''",")",">",">",">","annotation_read","=","``","wrd","''",">",">",">","bpe","=","SentencePiece","(","model_dir",",","100",",","annotation_train",",","annotation_read",",","model_type",",","annotation_format","=","'json","'",")",">",">",">","encoded_seq_ids",",","encoded_seq_pieces","=","bpe","(","...","batch_seq",",","batch_lens",",","dict_int2lab",",","task=","''","encode","''","...",")"]
docstr: Train tokenizer with unsupervised techniques (BPE, Unigram) using
  SentencePiece Library. If you use "char" mode, the SentencePiece
  creates a char dict so the vocab_size attribute is not needed.
docstr_tok: ["Train","tokenizer","with","unsupervised","techniques","(","BPE",",","Unigram",")","using","SentencePiece","Library",".","If","you","use","``","char","''","mode",",","the","SentencePiece","creates","a","char","dict","so","the","vocab_size","attribute","is","not","needed","."]
code: def _train_BPE(self):
      """Train tokenizer with unsupervised techniques (BPE, Unigram) using
      SentencePiece Library. If you use "char" mode, the SentencePiece
      creates a char dict so the vocab_size attribute is not needed.
      """
      query = (
          "--input="
          + self.text_file
          + " --model_prefix="
          + self.prefix_model_file
          + " --model_type="
          + self.model_type
          + " --bos_id="
          + self.bos_id
          + " --eos_id="
          + self.eos_id
          + " --pad_id="
          + self.pad_id
          + " --unk_id="
          + self.unk_id
          + " --max_sentencepiece_length="
          + self.max_sentencepiece_length
          + " --character_coverage="
          + self.character_coverage
          + " --add_dummy_prefix="
          + self.add_dummy_prefix
      )
      if self.model_type not in ["char"]:
          # include vocab_size
          query += " --vocab_size=" + str(self.vocab_size)
      if self.user_defined_symbols is not None:
          query += (
              " --user_defined_symbols=" + self.user_defined_symbols
          )
      if not self.split_by_whitespace:
          query += " --split_by_whitespace=false"
      # Train tokenizer
      spm.SentencePieceTrainer.train(query)
  
file_path: speechbrain/speechbrain/tokenizers/SentencePiece.py
imports: import os.path
  import torch
  import logging
  import csv
  import json
  import sentencepiece
  from speechbrain.dataio.dataio import merge_char
  from speechbrain.utils import edit_distance
  from speechbrain.utils.distributed import run_on_main
label_desc: Use image node_id 4 for calling the SentencePiece obj's underlying member method code with example usage: obj._train_BPE() without return types