class_docstr: BytePairEncoder trains byte pair encoding and
  encodes words using trained encoding
class_docstr_tok: ["BytePairEncoder","trains","byte","pair","encoding","and","encodes","words","using","trained","encoding"]
docstr: Creates byte pair encoder
  :param word_dictionary: A list of unique tokens ordered by order of merges during training (this is used when encoding).
  :type word_dictionary: list-like of type str.
  
  :param eow: special end-of-word token that will be added to each input word while encoding.
  :type eow: str.
  
  :param sow: special start-of-word token that will be added to each input word while encoding.
  :type sow: str.
  
  :param use_raw_bytes: treat strings as arrays of raw bytes (not utf-8).
  :type use_raw_bytes: bool, default=False.
  
  :param unknown_token_id: The id of the special 'unknown' token. All other tokens are enumerated from 'unknown_token_id' + 1.
  :type unknown_token_id: int, default=0.
docstr_tok: ["Creates","byte","pair","encoder",":","param","word_dictionary",":","A","list","of","unique","tokens","ordered","by","order","of","merges","during","training","(","this","is","used","when","encoding",")",".",":","type","word_dictionary",":","list-like","of","type","str",".",":","param","eow",":","special","end-of-word","token","that","will","be","added","to","each","input","word","while","encoding",".",":","type","eow",":","str",".",":","param","sow",":","special","start-of-word","token","that","will","be","added","to","each","input","word","while","encoding",".",":","type","sow",":","str",".",":","param","use_raw_bytes",":","treat","strings","as","arrays","of","raw","bytes","(","not","utf-8",")",".",":","type","use_raw_bytes",":","bool",",","default=False",".",":","param","unknown_token_id",":","The","id","of","the","special","'unknown","'","token",".","All","other","tokens","are","enumerated","from","'unknown_token_id","'","+","1.",":","type","unknown_token_id",":","int",",","default=0","."]
code: def __init__(
      self,
      word_dictionary: tp.List[str],
      *,
      eow: str,
      sow: str,
      use_raw_bytes: bool = False,
      unknown_token_id: int = 0,
  ):
      """Creates byte pair encoder
      :param word_dictionary: A list of unique tokens ordered by order of merges during training (this is used when encoding).
      :type word_dictionary: list-like of type str.
  
      :param eow: special end-of-word token that will be added to each input word while encoding.
      :type eow: str.
  
      :param sow: special start-of-word token that will be added to each input word while encoding.
      :type sow: str.
  
      :param use_raw_bytes: treat strings as arrays of raw bytes (not utf-8).
      :type use_raw_bytes: bool, default=False.
  
      :param unknown_token_id: The id of the special 'unknown' token. All other tokens are enumerated from 'unknown_token_id' + 1.
      :type unknown_token_id: int, default=0.
      """
      self._internal = PythonWrapper.BytePairEncoder()
  
      if word_dictionary is None:
          # Special mode for creating with 'train'/'load'. Not intended to use externally.
          pass
      else:
          self._internal.initialize(
              word_dictionary, eow, sow, use_raw_bytes, unknown_token_id
          )
  
file_path: neoml/NeoML/Python/neoml/BytePairEncoder.py
imports: import neoml.PythonWrapper
  import typing
label_desc: Use image node_id 1 to create a new BytePairEncoder object  with example: obj = BytePairEncoder(word_dictionary)