code: def __call__(
      self,
      text,
      return_tensors="pt",
      padding="max_length",
      truncation=True,
      add_bos=True,
  ):
      assert return_tensors == "pt"
      assert padding == "max_length"
  
      if isinstance(text, str):
          text = [text]
  
      ids = []
      for t in text:
          if t.startswith(f"{YaLMTokenizer.BOS_TOKEN} "):
              t_ids = self.tokenize(t[4:])
              t_ids = [1] + t_ids
          elif t.startswith(YaLMTokenizer.BOS_TOKEN):
              t_ids = self.tokenize(t[3:])
              t_ids = [1] + t_ids
          else:
              t_ids = self.tokenize(t)
              if add_bos:
                  t_ids = [1] + t_ids  # append <s>
  
          if truncation:
              if self.truncation_side == "left":
                  t_ids = t_ids[-self.model_max_length :]
              else:
                  t_ids = t_ids[: self.model_max_length]
  
          ids.append(t_ids)
  
      if padding != "max_length":
          max_len = max([len(t_ids) for t_ids in ids])
      else:
          max_len = self.model_max_length
  
      attention_mask = torch.ones(len(ids), max_len, dtype=torch.long)
  
      if self.padding_side == "left":
          new_ids = []
          for i, t_ids in enumerate(ids):
              attention_mask[i, : max_len - len(t_ids)] = 0
              new_ids.append(
                  [self.BOS] * (max_len - len(t_ids)) + t_ids
              )
      else:
          new_ids = []
          for i, t_ids in enumerate(ids):
              attention_mask[i, -(max_len - len(t_ids)) :] = 0
              new_ids.append(
                  t_ids + [self.EOS] * (max_len - len(t_ids))
              )
      ids = new_ids
      ids = torch.tensor(ids)
  
      if add_bos:
          # make sure starts with <s>
          ids[:, 0] = 1
  
      return {"input_ids": ids, "attention_mask": attention_mask}
  
imports: import importlib_resources
  import torch
  import sentencepiece
label_desc: Use image node_id 17 for calling the YaLMTokenizer obj's underlying member method code with example usage: obj.__call__(text, return_tensors, padding, truncation, add_bos) and returns: dict