code: def tokenize(self, line, out_type=int):
      line = convert_to_unicode(line)
      line = line.replace("\n", self.NEW_LINE)
  
      has_bos = False
      has_eos = False
  
      # Handle special tokens
      if line.startswith(f"{YaLMTokenizer.BOS_TOKEN} "):
          has_bos = True
          line = line[4:]
      elif line.startswith(YaLMTokenizer.BOS_TOKEN):
          has_bos = True
          line = line[3:]
      if line.endswith(f" {YaLMTokenizer.EOS_TOKEN}"):
          has_eos = True
          line = line[:-5]
      elif line.endswith(YaLMTokenizer.EOS_TOKEN):
          has_eos = True
          line = line[:-4]
  
      token_ids = self._encode(line, out_type=out_type)
      if has_bos:
          if out_type == int:
              token_ids = [1] + token_ids
          else:
              token_ids = [YaLMTokenizer.BOS_TOKEN] + token_ids
      if has_eos:
          if out_type == int:
              token_ids = token_ids + [2]
          else:
              token_ids = token_ids + [YaLMTokenizer.EOS_TOKEN]
      return token_ids
  
imports: import importlib_resources
  import torch
  import sentencepiece
label_desc: Use image node_id 3 for calling the YaLMTokenizer obj's underlying member method code with example usage: obj.tokenize(line, out_type) and returns: token_ids