code: def tokenize(self, tokenizer):
      self.tokenizer = tokenizer
      self._init_process()
      self._process_prompt()
  
      if self.pretraining:
          return
  
      self.source_ids = []
      source_ids = []
      bsz_num = math.ceil(len(self.source_text) / MAX_TOKENIZE_NUM)
      for i in range(bsz_num):
          ids = tokenizer(
              self.source_text[
                  i * MAX_TOKENIZE_NUM : (i + 1) * MAX_TOKENIZE_NUM
              ],
              add_special_tokens=False,
              return_token_type_ids=False,
              return_attention_mask=False,
          )["input_ids"]
          source_ids.extend(ids)
      for ids in source_ids:
          ids = (
              ids[: self.source_max_length]
              if self.config["truncate"] == "tail"
              else ids[-self.source_max_length :]
          )
          ids = self.prefix_ids + ids + self.suffix_ids
          if not self.is_casual_model:
              ids = self.tokenizer.build_inputs_with_special_tokens(ids)
          self.source_ids.append(torch.tensor(ids, dtype=torch.long))
  
      if self.paired_text and self.pretraining is None:
          self.target_ids = []
          target_ids = tokenizer(
              text_target=self.target_text,
              add_special_tokens=False,
              return_token_type_ids=False,
              return_attention_mask=False,
          )["input_ids"]
          for ids in target_ids:
              if self.config["truncate"] == "tail":
                  ids = ids[: self.target_max_length]
              else:
                  ids = ids[-self.target_max_length :]
              ids = self.tokenizer.build_inputs_with_special_tokens(ids)
              if self.config["model_name"] in [
                  "bert2bert",
                  "opt",
                  "unilm",
                  "xlm",
              ]:
                  ids = ids[1:]
              self.target_ids.append(
                  torch.tensor(ids, dtype=torch.long)
              )
  
file_path: TextBox/textbox/data/abstract_dataset.py
imports: import os
  import torch, warnings
  import math
  from torch.utils.data import DataLoader, Dataset
  from textbox import CLM_MODELS
  from typing import List
  from torch.nn.utils.rnn import pad_sequence
  from textbox.data.misc import load_data, _pad_sequence
label_desc: Use image node_id 6 for calling the AbstractDataset obj's underlying member method code with example usage: obj.tokenize(tokenizer) without return types