class_docstr: This will be superseded by a framework-agnostic approach soon.
class_docstr_tok: ["This","will","be","superseded","by","a","framework-agnostic","approach","soon","."]
code: def __init__(
      self,
      tokenizer: PreTrainedTokenizer,
      file_path: str,
      block_size: int,
      ref_path: str,
  ):
      warnings.warn(
          DEPRECATION_WARNING.format(
              "https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm_wwm.py"
          ),
          FutureWarning,
      )
      if os.path.isfile(file_path) is False:
          raise ValueError(f"Input file path {file_path} not found")
      if os.path.isfile(ref_path) is False:
          raise ValueError(f"Ref file path {file_path} not found")
      # Here, we do not cache the features, operating under the assumption
      # that we will soon use fast multithreaded tokenizers from the
      # `tokenizers` repo everywhere =)
      logger.info(f"Creating features from dataset file at {file_path}")
      logger.info(f"Use ref segment results at {ref_path}")
      with open(file_path, encoding="utf-8") as f:
          data = (
              f.readlines()
          )  # use this method to avoid delimiter '\u2029' to split a line
      data = [
          line.strip()
          for line in data
          if len(line) > 0 and not line.isspace()
      ]
      # Get ref inf from file
      with open(ref_path, encoding="utf-8") as f:
          ref = [
              json.loads(line)
              for line in f.read().splitlines()
              if (len(line) > 0 and not line.isspace())
          ]
      if len(data) != len(ref):
          raise ValueError(
              f"Length of Input file should be equal to Ref file. But the length of {file_path} is {len(data)} "
              f"while length of {ref_path} is {len(ref)}"
          )
  
      batch_encoding = tokenizer(
          data,
          add_special_tokens=True,
          truncation=True,
          max_length=block_size,
      )
      self.examples = batch_encoding["input_ids"]
      self.examples = [
          {"input_ids": torch.tensor(e, dtype=torch.long)}
          for e in self.examples
      ]
  
      n = len(self.examples)
      for i in range(n):
          self.examples[i]["chinese_ref"] = torch.tensor(
              ref[i], dtype=torch.long
          )
  
imports: import json
  import os
  import pickle
  import random
  import time
  import warnings
  from typing import Dict, List, Optional
  import torch
  from filelock import FileLock
  from torch.utils.data import Dataset
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 1 to create a new LineByLineWithRefDataset object from inherited base classes: Dataset with example: obj = LineByLineWithRefDataset(tokenizer, file_path, block_size, ref_path)